#!/bin/bash

#########################################################################
#precheck
check_value_is_not_null(){
    if [ ! -n "$1" ];then
        echo "the value is null"
        exit 1
    fi
}
#precheck
#########################################################################

#########################################################################
if [ -f /etc/ceph-cluster.env ];then
    source /etc/ceph-cluster.env
else
    echo "The ceph environment file /etc/ceph-cluster.env is not found!"
    exit 1
fi
OSD_WorkPath="/var/lib/ceph/osd/ceph-$OSD_ID"

if [ "$Deploy_Mode"x == binaryx ];then
    OSD_ID=$1
elif [ "$Deploy_Mode"x == dockerx ];then
    echo $OSD_ID
elif [ "$Deploy_Mode"x == k8sx ];then
    OSD_ID=`echo $HOSTNAME|awk -F'-' '{print $3}'`
    echo $OSD_ID
else
    echo "The Deploy_Mode is error!"
    exit 1
fi	

if echo $OSD_ID|egrep "^[0-9]+$";then
    echo "OSD_ID check done"
else
    echo "OSD_ID check error!"
fi
#
#########################################################################

#########################################################################
#for Deploy_Mode=binary
#source /etc/ceph/lib/init_ceph_lib.sh
#从第一个mon节点scp ceph.conf文件到当前节点
create_ceph_user(){
    if ! id ceph;then
        useradd -m -d /var/lib/ceph -g ceph -s /usr/sbin/nologin -c 'Ceph storage service'
    fi
}

create_ceph_confpath(){
    if [ ! -d /etc/ceph ];then
        mkdir -p /etc/ceph
    fi
    chown -R ceph:ceph /etc/ceph
}

create_ceph_workpath(){
    if [ ! -d /var/lib/ceph/osd/ceph-$OSD_ID ];then
        mkdir -p /var/lib/ceph/osd/ceph-$OSD_ID
    fi
    if [ ! -d /var/lib/ceph/bootstrap-osd ];then
        mkdir -p /var/lib/ceph/bootstrap-osd
    fi
    chown -R ceph:ceph /var/lib/ceph
}

chown_ceph_pwd(){
    chown -R ceph:ceph /var/lib/ceph
    chown -R ceph:ceph /etc/ceph
}

copy_osd_prestart_scripts(){
    if [ "$Deploy_Mode"x == binaryx ];then
        cp -f /etc/ceph/lib/ceph-osd-prestart.sh /usr/libexec/ceph/ceph-osd-prestart.sh
        chmod -R +x /usr/libexec/ceph
    elif [ "$Deploy_Mode"x == dockerx -o "$Deploy_Mode"x == k8sx ];then
        echo "Error: The ceph script file /usr/libexec/ceph/ceph-osd-prestart.sh is not found!"
        exit 1
    fi
}

get_ceph_conf(){
    if [ "$Deploy_Mode"x == binaryx ];then
        if [ -f /etc/ceph/$Cluster_Name.conf ];then
            echo "The file /etc/ceph/$Cluster_Name.conf already exists."
        else
            for ((i=0; 1<5; i++));do
                if scp -P $ssh_port  $Deploy_Node:/etc/ceph/$Cluster_Name.conf /etc/ceph/$Cluster_Name.conf && chown -R ceph:ceph /etc/ceph ;then
                    echo "第一个mon节点配置文件已生成！"
                    break
                else
                    echo "第一个mon节点配置文件未生成！等待10s后自动重试......"
                    sleep 10s
                fi
            done
        fi
    elif [ "$Deploy_Mode"x == dockerx -o "$Deploy_Mode"x == k8sx ];then
        if [ -f /etc/ceph/$Cluster_Name.conf ];then
            echo "Error: The ceph configure file /etc/ceph/$Cluster_Name.conf is not found!"
            exit 1
        fi
    fi
}

#从已存在的mon服务运行的节点scp client.admin.keyring文件到当前节点，并将属主和数组改为ceph用户ceph组（在扩容的mon节点执行）
get_client_admin_keyring() {
    if [ "$Deploy_Mode"x == binaryx ];then
        if [ -f /etc/ceph/$Cluster_Name.client.admin.keyring ];then
            echo "The file /etc/ceph/$Cluster_Name.client.admin.keyring already exists."
        else
            scp -P $ssh_port $Deploy_Node:/etc/ceph/$Cluster_Name.client.admin.keyring /etc/ceph/$Cluster_Name.client.admin.keyring && chown -R ceph:ceph /etc/ceph
        fi
    elif [ "$Deploy_Mode"x == dockerx -o "$Deploy_Mode"x == k8sx ];then
        if [ ! -f /etc/ceph/$Cluster_Name.client.admin.keyring ];then
            echo "Error: The ceph admin keyring file /etc/ceph/$Cluster_Name.client.admin.conf is not found!"
            exit 1
        fi
    fi
}

#从已存在的mon服务运行的节点scp client.bootstrap-osd keyring文件到当前节点，并将属主和数组改为ceph用户ceph组（在扩容的mon节点执行）
get_bootstrap_osd_keyring() {
    #ceph --cluster $Cluster_Name auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring && chown -R ceph:ceph /var/lib/ceph  #弃用，当ceph-mon master节点没起来之前改命令无法获取keyring，徒增等待时间
    if [ "$Deploy_Mode"x == binaryx ];then
        if [ -f /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring ];then
            echo "The file /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring already exists."
            if [ -f /etc/ceph/$Cluster_Name.keyring ];then
                echo "The file /etc/ceph/$Cluster_Name.keyring already exists."
            else
                cp -f /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring /etc/ceph/ && chown -R ceph:ceph /etc/ceph
            fi
        else
            scp -P $ssh_port $Deploy_Node:/var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring && chown -R ceph:ceph /var/lib/ceph
            cp -f /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring /etc/ceph/ && chown -R ceph:ceph /etc/ceph
        fi
    elif [ "$Deploy_Mode"x == dockerx -o "$Deploy_Mode"x == k8sx ];then
        if [ ! -f /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring ];then
            echo "Error: The ceph bootstrap-osd keyring file /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring is not found!"
            exit 1
        fi
    fi
}

bootstrap_osd_get_monmap(){
    ceph --cluster $Cluster_Name --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring mon getmap -o /var/lib/ceph/osd/$Cluster_Name-$OSD_ID/activate.monmap
    if [ "$?"x != 0x ];then
        echo "Error: get monmap failed!"
        exit 1
    fi
}

create_osd_keyring(){
    ceph auth get-or-create osd.$OSD_ID mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' -o /var/lib/ceph/osd/$Cluster_Name-$OSD_ID/keyring
    if [ "$?"x != 0x ];then
        echo "Error: create osd keyring failed!"
        exit 1
    fi
}

systemctl_daemon_reload(){
    systemctl daemon-reload
}

ceph_crash_achive_all(){
    if ! ceph crash archive-all;then
        echo "ceph crash archive-all is faild!"
    fi
}

load_bcache_module(){
    if ! modinfo bcache > /dev/null 2>&1 ;then
        touch /etc/modules-load.d/bcache.conf
        echo 'bcache' > /etc/modules-load.d/bcache.conf
        modprobe bcache
    fi
}

chown_osd_device(){
    Devices_Len=
    if [ "$Cache"x == yesx ];then
        echo "Cache_Disk:$Cache_Disk"
        echo "Data_Disk:$Data_Disk"
        echo "Block_DB_Part:$Block_DB_Part"
        echo "Block_WAL_Part:$Block_WAL_Part"
        Devices="$Data_Disk $Block_DB_Part $Block_WAL_Part $Cache_Disk"
        if [ "$Enable_BCache"x == yesx ];then
            echo "Block_BCache_Part:$Block_BCache_Part"
            echo "BCache_Device:$BCache_Device"
            Devices="$Devices $Block_BCache_Part $BCache_Device"
        fi
    else
        echo "Data_Disk:$Data_Disk"
        echo "Block_DB_Part:$Block_DB_Part"
        echo "Block_WAL_Part:$Block_WAL_Part"
        echo "Block_Data_Part:$Block_Data_Part"
        Devices="$Data_Disk $Block_DB_Part $Block_WAL_Part $Block_Data_Part"
    fi

    for i in $Devices;do
        if echo $i | egrep "hd*|nvme*|sd*|vd*|xvd*|bcache*|cciss*|dasd*|ubd*|ubi*|scm*|pmem*|nbd*|zd*";then
            if [ -b /dev/$i ];then
                chown -R ceph:ceph /dev/$i
                ls -l /dev/$i
            else
                echo $i && exit 1
            fi
        else
            echo $i && exit 1
        fi
    done
}

chown_osd_device_path(){
    Devices_Path_Len=
    if [ "$Cache"x == yesx ];then
        echo "Cache_Disk_Path:$Cache_Disk_Path"
        echo "Data_Disk_Path:$Data_Disk_Path"
        echo "Block_DB_Part_Path:$Block_DB_Part_Path"
        echo "Block_WAL_Part_Path:$Block_WAL_Part_Path"
        Devices_Path="$Data_Disk_Path $Block_DB_Part_Path $Block_WAL_Part_Path $Cache_Disk_Path"
        if [ "$Enable_BCache"x == yesx ];then
            echo "Block_BCache_Part_Path:$Block_BCache_Part_Path"
            Devices_Path="$Devices_Path $Block_BCache_Part_Path"
        fi
    else
        echo "Data_Disk_Path:$Data_Disk_Path"
        echo "Block_DB_Part_Path:$Block_DB_Part_Path"
        echo "Block_WAL_Part_Path:$Block_WAL_Part_Path"
        Devices_Path="$Data_Disk_Path $Block_DB_Part_Path $Block_WAL_Part_Path $Block_Data_Part_Path"
    fi
    for i in $Devices_Path;do
        if ls -l $i | egrep "hd*|nvme*|sd*|vd*|xvd*|bcache*|cciss*|dasd*|ubd*|ubi*|scm*|pmem*|nbd*|zd*";then
            if [ -L $i ];then
                chown -H ceph:ceph $i
                ls -l $i
            else
                echo $i && exit 1
            fi
        else
            echo $i && exit 1
        fi
    done
}

optimize_kernel_block_io(){
    if [ "$Enable_BCache"x == yesx ];then
        echo writeback > /sys/block/$BCache_Device/bcache/cache_mode
        echo 0 > /sys/block/$BCache_Device/bcache/sequential_cutoff
        echo 0 > /sys/block/$BCache_Device/bcache/cache/congested_read_threshold_us
        echo 0 > /sys/block/$BCache_Device/bcache/cache/congested_write_threshold_us
        echo fifo > /sys/block/$BCache_Device/bcache/cache/cache0/cache_replacement_policy
        echo 1 > /sys/block/$BCache_Device/bcache/cache/internal/copy_gc_enabled
        echo "8192" > /sys/block/$BCache_Device/queue/read_ahead_kb
        echo "40" > /sys/block/$BCache_Device/bcache/writeback_percent
    else
        echo 'Cache=yes Enable_BCache=no'
    fi

    if [ "$Cache"x == yesx ];then
        NR_REQUESTS=`cat /sys/block/$Cache_Disk/queue/nr_requests`
        if [ $NR_REQUESTS -lt 1024 ];then
            echo '1024' > /sys/block/$Cache_Disk/queue/nr_requests
        else
            echo nr_requests $NR_REQUESTS
        fi
        Cache_Scheduler=`cat /sys/block/$Cache_Disk/queue/scheduler |awk -F']' '{print $1}'|awk -F'[' '{print $NF}'`
        if [ "$Cache_Scheduler"x == nonex ];then
            echo Cache_Scheduler $Cache_Scheduler
        else
            echo 'none' > /sys/block/$Cache_Disk/queue/scheduler
        fi
        echo 'write through' > /sys/block/$Cache_Disk/queue/write_cache
        echo 'write through' > /sys/block/$Cache_Disk/device/scsi_disk/$Cache_Disk_SCSI_ID/cache_type
    fi
    echo 'write through' > /sys/block/$Data_Disk/queue/write_cache
    echo "8192" > /sys/block/$Data_Disk/queue/read_ahead_kb
    echo 'write through' > /sys/block/$Data_Disk/device/scsi_disk/$Data_Disk_SCSI_ID/cache_type
}
#global function
#########################################################################

#########################################################################
#start osd

link_block_data_to_osd_workpath(){
    if [ "$Enable_BCache"x == yesx ];then
        ceph-bluestore-tool prime-osd-dir --path $OSD_WorkPath --no-mon-config --dev /dev/$Block_BCache_Part
        if [ -h "$OSD_WorkPath/block" ];then
            echo "the block symbol exists"
        else
            #ln -s /dev/$BCache_Device $data/block
            ln -s /dev/$Data_Disk_Path $OSD_WorkPath/block
        fi
    else
        ceph-bluestore-tool prime-osd-dir --path $OSD_WorkPath --no-mon-config --dev /dev/$Block_Data_Part
        if [ -h "$OSD_WorkPath/block" ];then
            echo "the block symbol exists"
        else
            ln -s /dev/$Block_Data_Part_Path $OSD_WorkPath/block
        fi
    fi
}

link_block_db_to_osd_workpath(){
    if [ -h "$OSD_WorkPath/block.db" ];then
        echo "the block symbol exists"
    else
        ln -s $Block_DB_Part_Path $OSD_WorkPath/block.db
    fi
}

link_block_wal_to_osd_workpath(){
    if [ -h "$OSD_WorkPath/block.wal" ];then
        echo "the block symbol exists"
    else
        ln -s $Block_WAL_Part_Path $OSD_WorkPath/block.wal
    fi
}

mount_osd_pwd_for_systemd(){
    systemctl enable var-lib-ceph-osd-ceph\\x2d$OSD_ID.mount
    systemctl start var-lib-ceph-osd-ceph\\x2d$OSD_ID.mount
}

mount_osd_pwd_for_fstab(){
    mount -t tmpfs tmpfs /var/lib/ceph/osd/$Cluster_Name-$OSD_ID
}
#start osd
#########################################################################

#########################################################################
#creating osd workflow
request_new_osd() {
    OSD_UUID=`uuidgen`
    ceph --cluster $Cluster_Name --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/$Cluster_Name.keyring  osd new $OSD_UUID $OSD_ID
}

copy_mount_unit_to_systemd(){
    sed -i "s#^Where.*#Where=/var/lib/ceph/osd/$Cluster_Name-$OSD_ID#g" /etc/ceph/lib/ceph-osd.mount
    cp -f /etc/ceph/lib/ceph-osd.mount /usr/lib/systemd/system/var-lib-ceph-osd-ceph\\x2d$OSD_ID.mount
}

osd_mkfs(){
    #sleep 20s
    #check_env
    sudo -u ceph ceph-osd --cluster $Cluster_Name --conf /etc/ceph/$Cluster_Name.conf --osd-objectstore bluestore --mkfs -i $OSD_ID --name osd.$OSD_ID --monmap /var/lib/ceph/osd/$Cluster_Name-$OSD_ID/activate.monmap --keyring /var/lib/ceph/osd/$Cluster_Name-$OSD_ID/keyring --bluestore-block-wal-path $Block_WAL_Part_Path --bluestore-block-db-path $Block_DB_Part_Path --osd-data /var/lib/ceph/osd/$Cluster_Name-$OSD_ID --osd-uuid $OSD_UUID --setuser ceph --setgroup ceph --debug_osd 10 --debug_ms 10
    if [ "$?"x == 0x ];then
        if cat /var/lib/ceph/osd-env/osd.$OSD_ID |grep -w 'OSD_Status';then
            sed -i "/OSD_Status/d" /var/lib/ceph/osd-env/osd.$OSD_ID
        fi
        echo 'OSD_Status=ready' >> /var/lib/ceph/osd-env/osd.$OSD_ID
    else
        echo "osd mkfs faild!"
        exit 1
    fi
}

check_osd_data_disk(){
    if [ "$Enable_BCache"x == "yes"x ];then
        ceph-bluestore-tool show-label --dev /dev/$BCache_Device
    elif [ "$Cache"x == "yes" ];then
        ceph-bluestore-tool show-label --dev $Data_Disk_Path
    else
        ceph-bluestore-tool show-label --dev $Block_Data_Part_Path
    fi
}

add_env_file_to_osd_systemd_unit(){
    if ! grep -w 'osd-env' /usr/lib/systemd/system/$Cluster_Name-osd@.service;then
        sed -i '/^EnvironmentFile=-\/etc\/default\/ceph/a\EnvironmentFile=-\/var\/lib\/ceph\/osd-env\/osd./%i' /usr/lib/systemd/system/$Cluster_Name-osd@.service
    fi
}

#启动ceph-osd服务，并设置服务自启动
start_osd_service() {
    if [ "$Cluster_Name"x == "ceph"x ];then
        systemctl daemon-reload && systemctl start ceph-osd@$OSD_ID && systemctl enable ceph-osd@$OSD_ID
    else
        systemctl daemon-reload && systemctl start $Cluster_Name-osd@$OSD_ID && systemctl enable $Cluster_Name-osd@$OSD_ID
    fi
}

make_bcache(){
    partprobe & wait

    if [ -z "$Block_BCache_Part_Path" ];then
        echo "Block_BCache_Part_Path is null!"
        exit 1
    else
        while lsof $Block_BCache_Part_Path;do
            echo "$Block_BCache_Part_Path device is busy!"
            sleep 2
        done & wait
    fi
    if [ -z "$Data_Disk_Path" ];then
        echo "Data_Disk_Path is null!"
        exit 1
    else
        while lsof $Data_Disk_Path;do
            echo "$Data_Disk_Path device is busy!"
            sleep 2
        done & wait
    fi

    make-bcache -B $Data_Disk_Path -C $Block_BCache_Part_Path -w $BCache_Block_Size -b $BCache_Bucket_Size --writeback --wipe-bcache
    if [ "$?" != "0" ];then
        echo "make bcache faild!  $Data_Disk_Path $Block_BCache_Part_Path"
    else
        BCache_Device_UUID=`bcache-super-show $Data_Disk_Path|grep -w 'dev.uuid'|awk '{print $NF}'`
        BCache_Device_by_uuid="/dev/bcache/by-uuid/$BCache_Device_UUID"
        BCache_Cache_UUID=`bcache-super-show $Block_BCache_Part_Path|grep -w 'dev.uuid'|awk '{print $NF}'`
        BCache_Data_UUID=`bcache-super-show $Data_Disk_Path|grep -w 'dev.uuid'|awk '{print $NF}'`
        BCache_Cset_UUID=`bcache-super-show $Data_Disk_Path|grep -w 'cset.uuid'|awk '{print $NF}'`
        #echo "BCache_Device_UUID=$BCache_Device_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID
        echo "BCache_Cache_UUID=$BCache_Cache_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID
        echo "BCache_Data_UUID=$BCache_Data_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID
        echo "BCache_Cset_UUID=$BCache_Cset_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID
    fi
    while [ ! -L /dev/bcache/by-uuid/$BCache_Device_UUID ];do
        echo 'Waiting for the bcache device to be ready......'
        sleep 1s
    done
}

attach_bcache(){
    BCache_Backing_State=`bcache-super-show $Data_Disk_Path|grep -w 'dev.data.cache_state'|awk -F'[' '{print $2}'|awk -F']' '{print $1}'`
    if [ -z "$BCache_Backing_State" ];then
        echo "The value BCache_Backing_State is null"
        exit 1
    else
        if [ "$BCache_Backing_State"x == 'detachedx' ];then
            echo "$Data_Disk" > /sys/fs/bcache/register
        fi
    fi
}

wipe_osd_device(){
    #warning! The partition table must be reloaded in order for lsof to find the partition.
    partprobe & wait
    if [ -z $OSD_Device ];then
        echo "OSD_Device is null!"
        exit 1
    else
        while lsof $OSD_Device;do
            echo "$OSD_Device device is busy!"
            sleep 2
        done & wait
    fi

    if [ -b $OSD_Device ];then
        #wipefs -a $OSD_Device && sync 废弃
        dd if=/dev/zero of=$OSD_Device bs=4M count=8 oflag=dsync
    else
        echo "OSD_Device $OSD_Device device not found!"
        exit 1
    fi

    partprobe & wait
    lsblk
}

pre_check_disk(){
    Disk_Info=`parted -s $mkPart_Disk_Path unit s print 2>&1`
    Disk_Part_Info=`echo "$Disk_Info"|grep -w 'primary'`

    #判断当前硬盘是否是未创建分区表的空盘
    if echo "$Disk_Part_Info"|grep -w "Error: /dev/.*: unrecognised disk label";then
        #判断当前即将创建的硬盘分区是否是当前硬盘第一个分区
        if [ "$OSD_Status"x == readyx ];then
            echo "The Disk Part is error! Partition location error."
        else
            #清空缓存盘分区表
            OSD_Device="/dev/$mkPart_Disk_Path"
            wipe_osd_device
            #重新加载硬盘分区表
            #修改设备属主属组为ceph:ceph
            #chown -h ceph:ceph $OSD_Device
            #为当前缓存盘创建gpt分区表
            parted -s $OSD_Device mklabel gpt 2>&1
            cache_disk_number=$(($cache_disk_number + 1))
        fi
    fi
}

pre_check_part(){
    Current_Part_Start_Sector=$Part_Start_Sector
    Current_Part_End_Sector=$Part_End_Sector
    Disk_Info=`parted -s $mkPart_Disk_Path unit s print 2>&1`
    Disk_Part_Info=`echo "$Disk_Info"|egrep -w "^[[:space:]][0-9]+[[:space:]]+[0-9]+s+[[:space:]]+[0-9]+s+[[:space:]]+[0-9]+s"`
    Parts_Start_Sector=`echo "$Disk_Part_Info"|awk '{print $2}'|awk -F's' '{print $1}'|tr -s "\r\n" " "`
    Parts_End_Sector=`echo "$Disk_Part_Info"|awk '{print $3}'|awk -F's' '{print $1}'|tr -s "\r\n" " "`
    Parts_Used_Sectors="$Parts_Start_Sector $Parts_End_Sector"
    #检查现有分区的起始和结束扇区号是否有大于目标分区的起始扇区号，如果有，则表示分区空间已被占用
    for i in $Parts_Used_Sectors;do
        if [ "$Current_Part_Start_Sector" -le "$i" ];then
            echo "The Part_Start_Sector:$Current_Part_Start_Sector is in used!"
            exit 1
        fi
    done

    #判断当前硬盘是否是未创建分区表的空盘
    if echo "$Disk_Part_Info"|grep -w "Error: /dev/.*: unrecognised disk label";then
        #判断当前即将创建的硬盘分区是否是当前硬盘第一个分区
        if [ "$Current_Part_Start_Sector"x == "$Start_Sector"x ];then
            echo "The Disk Part check done."
        else
            echo "The Disk Part is error! Partition location error."
            echo "Current_Part_Start_Sector=$Current_Part_Start_Sector"
            echo "Current_Part_End_Sector=$Current_Part_End_Sector"
            exit 1

        fi
        #dd if=/dev/zero of=$mkPart_Disk_Path bs=4M count=1 oflag=dsync
        #清空缓存盘分区表
        OSD_Device="/dev/$Cache_Disk"
        wipe_osd_device
        #重新加载硬盘分区表
        #修改设备属主属组为ceph:ceph
        #chown -h ceph:ceph $OSD_Device
        #为当前缓存盘创建gpt分区表
        parted -s $OSD_Device mklabel gpt 2>&1
        cache_disk_number=$(($cache_disk_number + 1))
        #
        #mkPart_Disk_Path="$Cache_Device_by_path"

        Last_Part_Number=0
        Part_Number=$Last_Part_Number
    else
        Last_Part_Info=`echo "$Disk_Part_Info" |tail -n1`
        Last_Part_Number=`echo "$Last_Part_Info" |awk '{print $1'`
        Last_Part_Start_Sector=`echo "$Last_Part_Info" |awk '{print $2}'|awk -F's' '{print $1}'`
        Last_Part_End_Sector=`echo "$Last_Part_Info" |awk '{print $3}'|awk -F's' '{print $1}'`
        Last_Part_Sectors=`echo "$Last_Part_Info" |awk '{print $4}'|awk -F's' '{print $1}'`
        Last_Part_End_Let_Sector=$((Current_Part_Start_Sector - 1))
        if [ "$Last_Part_End_Sector"x == "$Last_Part_End_Let_Sector"x ];then
            echo "The Disk Part check done."
        elif [ "$Last_Part_End_Sector" -gt "$Last_Part_End_Let_Sector" ];then
            echo "The Disk Part is error! Partition location error."
            exit 1
        elif [ "$Last_Part_End_Sector" -lt "$Last_Part_End_Let_Sector" ];then
            while [ "$Last_Part_End_Sector" -eq "$Last_Part_End_Let_Sector" ];do
                echo "Waiting for the previous partition to be created......"
                Disk_Info=`parted -s $mkPart_Disk_Path unit s print 2>&1`
                Disk_Part_Info=`echo "$Disk_Info"|grep -w 'primary'`
                Last_Part_End_Sector=`echo "$Last_Part_Info" |awk '{print $3}'|awk -F's' '{print $1}'`
            done
        else
            echo "Unknown error, please check the values of the variables."
            echo "Current_Part_Start_Sector=$Current_Part_Start_Sector"
            echo "Current_Part_End_Sector=$Current_Part_End_Sector"
            echo "Last_Part_Info=$Last_Part_Info"
            echo "Last_Part_Number=$Last_Part_Number"
            echo "Last_Part_Start_Sector=$Last_Part_Start_Sector"
            echo "Last_Part_End_Sector=$Last_Part_End_Sector"
            echo "Last_Part_Sectors=$Last_Part_Sectors"
            echo "Last_Part_End_Let_Sector=$Last_Part_End_Let_Sector"
            echo "Disk_Info=$Disk_Info"
            echo "Disk_Part_Info=$Disk_Part_Info"
        fi
        Part_Number=$Last_Part_Number
    fi
}

create_disk_part(){
    partprobe & wait
    if [ -z "$mkPart_Disk_Path" ];then
        echo "mkPart_Disk_Path $mkPart_Disk_Path is null!"
        exit 1
    else
        #检查待分区设备是否被占用
        while lsof $mkPart_Disk_Path;do
            sleep 2
        done & wait
        #分区起始位置8M空间数据置零，预防该位置已有某些文件系统时触发udev事件，导致分区被busy
        #注意：此处parted的单位换算是按1000*1000计算，不是1024*1024计算，所以seek的单位必须是M，不能是MB,SEEK也要按照1000计算
        SEEK=$((Part_Start_Sector * 512))
        if [ -b $mkPart_Disk_Path ];then
            dd if=/dev/zero of=$mkPart_Disk_Path bs=4MB count=4 seek="$SEEK"b oflag=dsync
            case $END in
                100%)
                    parted -s $mkPart_Disk_Path mkpart $Part_Volume_Name "$Part_Start_Sector"s 100% && sync
                    if [ "$?" != 0 ];then
                        echo "Error: create partition failed!"
                        exit 1
                    fi
                    ;;
                [0-9]*)
                    parted -s $mkPart_Disk_Path mkpart $Part_Name "$Part_Start_Sector"s "$Part_End_Sector"s && sync
                    if [ "$?" != 0 ];then
                        echo "Error: create partition failed!"
                        exit 1
                    fi
                    ;;
                *)
                    echo "create disk part error! END:$Part_End_Sector"
                    exit 1
                    ;;
            esac
        else
            echo "mkPart_Disk_Path $mkPart_Disk_Path is not block device!"
            exit 1
        fi
    fi
    unset mkPart_Disk_Path
    unset Part_Start_Sector
    unset Part_End_Sector
}

get_part_info(){
    Part_Number=`parted -s /dev/$Disk print |grep -w "$Part_Name"|awk '{print $1}'
    Part_Name="$Disk$Part_Number"
    Part_By_Path="/dev/disk/by-path`ls -l /dev/disk/by-path|grep -w "$Part_Name$"|awk '{print $9}'`"
    Part_UUID=`blkid /dev/$Part_Name|awk '{print $NF}'|awk -F'"' '{print $2}'`
}

update_block_db_path(){
    Env_Block_DB_Path=`cat /var/lib/ceph/osd-env/osd.$OSD_ID|egrep -v "^#|^$"|grep -w "^Block_DB_Part_Path="|awk -F'=' '{print $2}'`
    if [ "$Part_By_Path"x != "$Env_Block_DB_Path" ];then
        sed -i  "/Block_DB_Part_Path=/d" /var/lib/ceph/osd-env/osd.$OSD_ID
        echo "Block_DB_Part_Path=$Part_By_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
    fi
    echo "Block_DB_Part_UUID=$Part_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID
}

update_block_wal_path(){
    Env_Block_WAL_Path=`cat /var/lib/ceph/osd-env/osd.$OSD_ID|egrep -v "^#|^$"|grep -w "^Block_WAL_Part_Path="|awk -F'=' '{print $2}'`
    if [ "$Part_By_Path"x != "$Env_Block_WAL_Path" ];then
        sed -i  "/Block_WAL_Part_Path=/d" /var/lib/ceph/osd-env/osd.$OSD_ID
        echo "Block_WAL_Part_Path=$Part_By_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
    fi
    echo "Block_WAL_Part_UUID=$Part_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID
}

update_block_data_path(){
    Env_Block_Data_Path=`cat /var/lib/ceph/osd-env/osd.$OSD_ID|egrep -v "^#|^$"|grep -w "^Block_Data_Part_Path="|awk -F'=' '{print $2}'`
    if [ "$Part_By_Path"x != "$Env_Block_Data_Path" ];then
        sed -i  "/Block_Data_Part_Path=/d" /var/lib/ceph/osd-env/osd.$OSD_ID
        echo "Block_Data_Part_Path=$Part_By_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
    fi
    echo "Block_Data_Part_UUID=$Part_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID
}

update_block_bcache_path(){
    Env_Block_BCache_Path=`cat /var/lib/ceph/osd-env/osd.$OSD_ID|egrep -v "^#|^$"|grep -w "^Block_BCache_Part_Path="|awk -F'=' '{print $2}'`
    if [ "$Part_By_Path"x != "$Env_Block_BCache_Path" ];then
        sed -i  "/Block_BCache_Part_Path=/d" /var/lib/ceph/osd-env/osd.$OSD_ID
        echo "Block_BCache_Part_Path=$Part_By_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
    fi
    echo "Block_BCache_Part_UUID=$Part_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID
}

disk_mkpart_stack() {
    if [ "$Cache"x == yesx ];then
        Disk=$Cache_Disk

        mkPart_Disk_Path=$Cache_Disk_Path
        Part_Start_Sector=$Block_DB_Part_Start_Sector
        Part_End_Sector=$Block_DB_Part_End_Sector
        pre_check_part
        Part_Volume_Name=osd_"$OSD_ID"_db
        create_disk_part
        get_part_info
        update_block_db_path

        mkPart_Disk_Path=$Cache_Disk_Path
        Part_Start_Sector=$Block_WAL_Part_Start_Sector
        Part_End_Sector=$Block_WAL_Part_End_Sector
        pre_check_part
        Part_Volume_Name=osd_"$OSD_ID"_wal
        create_disk_part
        get_part_info
        update_block_wal_path

        if [ "$Enable_BCache"x == yesx ];then
            mkPart_Disk_Path=$Cache_Disk_Path
            Part_Start_Sector=$Block_DB_Part_Start_Sector
            Part_End_Sector=$Block_DB_Part_End_Sector
            pre_check_part
            Part_Volume_Name=osd_"$OSD_ID"_bcache
            create_disk_part
            get_part_info
            update_block_bcache_path
        fi
    else
        mkPart_Disk_Path=$Data_Disk_Path
        Part_Start_Sector=$Block_DB_Part_Start_Sector
        Part_End_Sector=$Block_DB_Part_End_Sector
        pre_check_part
        Part_Volume_Name=osd_"$OSD_ID"_db
        create_disk_part
        get_part_info
        update_block_db_path

        mkPart_Disk_Path=$Data_Disk_Path
        Part_Start_Sector=$Block_WAL_Part_Start_Sector
        Part_End_Sector=$Block_WAL_Part_End_Sector
        pre_check_part
        Part_Volume_Name=osd_"$OSD_ID"_wal
        create_disk_part
        get_part_info
        update_block_wal_path

        mkPart_Disk_Path=$Data_Disk_Path
        Part_Start_Sector=$Block_Data_Part_Start_Sector
        Part_End_Sector=$Block_Data_Part_End_Sector
        pre_check_part
        Part_Volume_Name=osd_"$OSD_ID"_data
        create_disk_part
        get_part_info
        update_block_data_path
    fi
    #
    sleep 10s
    #systemctl restart systemd-udevd
    sleep 15s
}

#################
#environment generate
gen_osd_id_range(){
    #Node_Number=`hostname -s|awk -F'-' '{print $2}'`
    let tmp_number=1
    check_value_is_not_null $osd_nodes
    check_value_is_not_null $HostName
    check_value_is_not_null $Node_OSDs
    for i in $osd_nodes;do
        echo "i=$i  HostName=$HostName"
        if [ "$HostName"x == "$i"x ];then
            echo "Node_Number=$tmp_number"
            Node_Number=$(($tmp_number))
            OSD_ID_End=$(($Node_Number * $Node_OSDs - 1))
            OSD_ID_Start="$(($OSD_ID_End + 1 - $Node_OSDs))"
            echo "OSD_ID_Start=$OSD_ID_Start"
            break
        else
            tmp_number=$(($tmp_number + 1))
        fi
    done
}

mkdir_osd_mapping(){
    if [ -d /var/lib/ceph/osd-env ];then
        echo "The dirctor /var/lib/ceph/osd-env is already exist!"
    else
        mkdir /var/lib/ceph/osd-env
    fi
}

gen_disk_by_path_map(){
    if [ ! -f /etc/ceph/ceph-osd.map ];then
        echo "Then file ceph-osd.map is not found."
        exit 1
    fi
    gen_osd_id_range
    mkdir_osd_mapping
    #初始化osd id
    check_value_is_not_null "$OSD_ID_Start"
    osd_id="$OSD_ID_Start"
    echo "osd_id=$OSD_ID_Start"
    check_value_is_not_null "$OSD_ID_End"

    #获取$Cluster_Name-osd.map当前节点上缓存盘列表
    Local_Devices=`cat /etc/ceph/ceph-osd.map|egrep -v "^$|^#" |grep -w "$Node_FQDN"`
    check_value_is_not_null "$Local_Devices"
    Local_Cache_Devices_Map=`echo "$Local_Devices"|awk '{print $2}'`
    check_value_is_not_null "$Local_Cache_Devices_Map"
    echo "Local_Cache_Devices_Map $Local_Cache_Devices_Map"
    caches=0
    fusion_disks=0
    for i in $Local_Cache_Devices_Map;do
        if echo $i|grep ':';then
            if [ "$Cache"x == yesx ];then
                caches=$((caches+1))
            elif [ "$fusion_disks"x = 0x ] ;then
                Cache=yes
                caches=$((caches+1))
            else
                echo "cache disk $i is not found!" 
                exit 1
            fi
        else
            if [ "$Cache"x == nox ];then
                fusion_disks=$((fusion_disks+1))
            elif [ "$caches"x == 0x ];then
                Cache=no
                fusion_disks=$((fusion_disks+1))
                Enable_BCache=no
            else
                echo "Discovering cache disk in fusion disk deployment mode. cache disk is $i!"  
                exit 1
            fi
        fi
    done
    if [ "$Cache"x == yesx ];then
        Local_Cache_Devices=`echo "$Local_Cache_Devices_Map"|awk -F':' '{print $1}'|sort -u`
        check_value_is_not_null "$Local_Cache_Devices"
        for OSD_Cache_Device in $Local_Cache_Devices;do
            Cache_Disk_Path="/dev/disk/by-path/`ls -l /dev/disk/by-path/|egrep -w "$OSD_Cache_Device$"|awk '{print $9}'`"
            OSD_Data_Devices=`echo "$Local_Devices"|egrep  "^$Node_FQDN $OSD_Cache_Device:"|awk -F':' '{print $NF}'|tr -s "\r\n" " "`
            data_number=0
            check_value_is_not_null "$Cache_Disk_Path"
            check_value_is_not_null "$OSD_Data_Devices"
            for Data_Disk in $OSD_Data_Devices;do
                data_number=$((data_number + 1))
                Data_Disk_Path="/dev/disk/by-path/`ls -l /dev/disk/by-path/|egrep -w "$Data_Disk$"|awk '{print $9}'`"
                OSD_Values="$OSD_Values \n$Node_FQDN ceph-$osd_id $data_number $Data_Disk_Path $Cache_Disk_Path"
                check_value_is_not_null "$Data_Disk_Path"
                check_value_is_not_null "$OSD_Values"
                echo "$osd_id $OSD_ID_Start $osd_id $OSD_ID_End"
                if [ "$osd_id" -lt "$OSD_ID_Start" -a "$osd_id" -gt "$OSD_ID_End" ];then
                    echo "osd_id:$osd_id is error."
                    exit 1
                fi
                osd_id=$(($osd_id + 1))
            done
        done
    else
        OSD_Data_Devices=`echo "$Local_Devices"|awk '{print $1=null,$0}'|tr -s "\r\n" " "`
        check_value_is_not_null "$OSD_Data_Devices"
        for Data_Disk in $OSD_Data_Devices;do
            data_number=$((data_number + 1))
            Data_Disk_Path="/dev/disk/by-path/`ls -l /dev/disk/by-path/|egrep -w "$Data_Disk$"|awk '{print $9}'`"
            OSD_Values="$OSD_Values \n$Node_FQDN ceph-$osd_id $data_number $Data_Disk_Path"
            if [ "$osd_id" -lt "$OSD_ID_Start" -a "$osd_id" -gt "$OSD_ID_End" ];then
                echo "osd_id:$osd_id is error."
            fi
            osd_id=$(($osd_id + 1))
        done
    fi
    echo -e "$OSD_Values" > /etc/ceph/ceph-osd-disk-path.map
}

disk_path_map_file_check(){
    if [ -f /etc/ceph/ceph-osd-disk-path.map ];then
        echo "Than file /etc/ceph/ceph-osd-disk-path.map is already exists."
    else
        if [ "$Deploy_Mode"x == k8sx ];then
            while [ ! -f /etc/ceph/ceph-osd-disk-path.map ];do
                echo "Than file /etc/ceph/ceph-osd-disk-path.map is not found."
                sleep 1s
            done
        else
            gen_disk_by_path_map
        fi
    fi
}

cache_osd_env(){
cat > /var/lib/ceph/osd-env/ceph-osd.$OSD_ID <<- EOF
#!/bin/bash
Cache=yes
Enable_BCache=$Enable_BCache
OSD_ID=$OSD_ID
Ceph_FSID=$Ceph_FSID
Block_WAL_Part_Start_Sector=$Block_WAL_Part_Start_Sector
Block_WAL_Part_End_Sector=$Block_WAL_Part_End_Sector
Block_DB_Part_Start_Sector=$Block_DB_Part_Start_Sector
Block_DB_Part_End_Sector=$Block_DB_Part_End_Sector
Block_BCache_Part_Start_Sector=$Block_BCache_Part_Start_Sector
Block_BCache_Part_End_Sector=$Block_BCache_Part_End_Sector
Data_Disk_Path='$Data_Disk_Path' 
Block_DB_Part_Path='$Block_DB_Part_Path' 
Block_WAL_Part_Path='$Block_WAL_Part_Path' 
Block_BCache_Part_Path='$Block_BCache_Part_Path' 
Cache_Disk_Path='$Cache_Disk_Path' 
Cache_Disk_SCSI_ID='$Cache_Disk_SCSI_ID' 
Data_Disk_SCSI_ID='$Data_Disk_SCSI_ID'
EOF
}

fusion_osd_env(){
cat > /var/lib/ceph/osd-env/ceph-osd.$OSD_ID <<- EOF
#!/bin/bash
Cache=no
Enable_BCache=no
OSD_ID=$OSD_ID
Ceph_FSID=$Ceph_FSID
Block_DB_Part_Start=$Block_DB_Part_Start
Block_DB_Part_End=$Block_DB_Part_End
Block_WAL_Part_Start=$Block_WAL_Part_Start
Block_WAL_Part_End=$Block_WAL_Part_End
Block_Data_Part_Start=$Block_Data_Part_Start
Block_Data_Part_End=$Block_Data_Part_End
Data_Disk_Path='$Data_Device_Path'
Block_DB_Part_Path='$Block_DB_Part_Path'
Block_WAL_Part_Path='$Block_WAL_Part_Path'
Block_Data_Part_Path='$Block_Data_Part_Path'
Data_Disk_SCSI_ID='$Data_Disk_SCSI_ID'
EOF
}

Sector_Optimal(){
    Optimal_IO_Size=`cat /sys/block/$Pre_mkpart_Disk/queue/optimal_io_size`
    Minimum_IO_Size=`cat /sys/block/$Pre_mkpart_Disk/queue/minimum_io_size`
    Physical_Block_Size=`cat /sys/block/$Pre_mkpart_Disk/queue/physical_block_size`
    Logical_Block_Size=`cat /sys/block/$Pre_mkpart_Disk/queue/logical_block_size`
    Alignment_Offset=`cat /sys/block/$Pre_mkpart_Disk/alignment_offset`
    check_value_is_not_null "$Pre_mkpart_Disk"
    check_value_is_not_null "$Optimal_IO_Size"
    check_value_is_not_null "$Minimum_IO_Size"
    check_value_is_not_null "$Physical_Block_Size"
    check_value_is_not_null "$Logical_Block_Size"
    check_value_is_not_null "$Alignment_Offset"
    #计算物理扇区是否是逻辑扇区整数倍
    echo "Physical_Block_Size=$Physical_Block_Size Logical_Block_Size=$Logical_Block_Size"
    if expr "$Physical_Block_Size" % "$Logical_Block_Size" ;then
        echo "Physical_Block_Size % Logical_Block_Size is not null!"
        exit 1
    fi
    #计算最优起始扇区号
    if [ "$Optimal_IO_Size"x == 0x ];then
        if [ "$Minimum_IO_Size"x == 0x ];then
            Start_Sector=$((Physical_Block_Size/Logical_Block_Size))
        else
            Start_Sector=$(((Physical_Block_Size+Alignment_Offset)/Logical_Block_Size))
        fi
        Start_Sector=$((Start_Sector*2048))
        check_value_is_not_null "$Start_Sector"
    else
        Start_Sector=$(((Optimal_IO_Size+Alignment_Offset)/Physical_Block_Size))
    fi
    #计算逻辑扇区映射到物理扇区的步长
    Sector_Step_Size=$((Physical_Block_Size/Logical_Block_Size))
    Next_Part_Start_Sector=$Start_Sector
}

Sector_let(){
    Part_Start_Sector=$Next_Part_Start_Sector
    check_value_is_not_null "$Part_Start_Sector"
    check_value_is_not_null "$Part_Size"
    if [ "$Part_Size"x == '100%x' ];then
        Part_End_Sector='100%'
    else
        #计算分区容量，以Byte为单位
        Part_Size_Byte=$((Part_Size * 1024 * 1024 * 1024))
        #通过分区容量求出分区理论扇区数量
        Capacity_End_Sectors=$((Part_Size_Byte / Logical_Block_Size))
        if expr $Capacity_End_Sectors % $Sector_Step_Size ;then
            #Remander_Sectors=$((Capacity_End_Sectors % Sector_Step_Size))
            #Difference_Sectors=$((Sector_Step_Size - Remander_Sectors))
            #Part_End_Sectors=$((Capacity_End_Sectors + Difference_Sectors))
            Part_End_Sector=$((Sector_Step_Size - (Capacity_End_Sectors % Sector_Step_Size) + Capacity_End_Sectors + Part_Start_Sector))
        else
            Part_End_Sector=$((Capacity_End_Sectors + Part_Start_Sector))
        fi
        Next_Part_Start_Sector=$((Part_End_Sector + 1))
    fi
}

gen_osd_env() {
    disk_path_map_file_check
    Local_OSD_Values=`cat /etc/ceph/ceph-osd-disk-path.map|egrep -v "^#|^$"|grep -w "^$Node_FQDN"`
    Cache_Devices=`echo "$Local_OSD_Values"|awk '{print $5}'`
    if echo "$Cache_Devices" |grep '/dev/disk/by-path/pci-';then
        Cache=yes
    else
        Cache=no
    fi
    if [ "$Cache"x == nox ];then
        #Fusion Deployment Model Based on All-Flash or All-HDD
        OSD_Fusion_Devices=`echo "$Local_OSD_Values"|awk '{print $4}'`
        Fusion_Device_Values=`echo "$Local_OSD_Values"|grep -w "ceph-$OSD_ID"`
        Data_Disk_Path=`echo "$Fusion_Device_Values"|awk '{print $4}'`
        Data_Disk=`ls -l $Data_Disk_Path|awk -F'/' '{print $NF}'`
        Fusion_Disk_Number=`echo "$Fusion_Device_Values"|awk '{print $3}'`

        Pre_mkpart_Disk=$Data_Disk
        Sector_Optimal
        
	Part_Number=0

        Part_Size="$Block_DB_Size"
        Sector_let
        Part_Number=$((Part_Number + 1))
        Block_DB_Part_Path=$Data_Disk_Path-part$Part_Number
        Block_DB_Part="$Data_Disk$Part_Number"
        Block_DB_Part_Start_Sector="$Part_Start_Sector"
        Block_DB_Part_End_Sector="$Part_End_Sector"

        Part_Size="$Block_WAL_Size"
        Sector_let
        Part_Number=$((Part_Number + 1))
        Block_WAL_Part_Path=$Data_Disk_Path-part$Part_Number
        Block_WAL_Part="$Data_Disk$Part_Number"
        Block_WAL_Part_Start_Sector="$Part_Start_Sector"
        Block_WAL_Part_End_Sector="$Part_End_Sector"

        Part_Size='100%'
        Sector_let
        Part_Number=$((Part_Number + 1))
        Block_Data_Part_Path=$Data_Disk_Path-part$Part_Number
        Block_Data_Part="$Data_Disk$Part_Number"
        Block_Data_Part_Start_Sector="$Part_Start_Sector"
        Block_Data_Part_End_Sector="$Part_End_Sector"

        #生成osd disk SCSI相关变量
        Data_Disk_SCSI_ID=`ls -l /sys/block/$Data_Disk/device|awk -F'/' '{print $NF}'`
        #将osd映射表重定向至文件
        fusion_osd_env
    elif [ "$Cache"x == yesx ];then
        #Mixed Deployment Model Based on NVMe, SSD, and HDD
        Data_Device_Values=`echo "$Local_OSD_Values"|grep -w "ceph-$OSD_ID"`
        Cache_Disk_Path=`echo $Data_Device_Values|awk '{print $5}'`
        Data_Disk_Path=`echo "$Data_Device_Values"|awk '{print $4}'`
        Data_Disk_Number=`echo $Data_Device_Values|awk '{print $3}'`
        echo "Data_Disk_Number=$Data_Disk_Number"
        #Data_Disks_Path=`echo "$Local_OSD_Values"|grep -w "$Cache_Disk_Path"|awk '{print $4}'`
        Cache_Disk=`ls -l $Cache_Disk_Path|awk -F'/' '{print $NF}'`
        Data_Disk=`ls -l $Data_Disk_Path|awk -F'/' '{print $NF}'`
        
        #
        if [ "$Data_Disk_Number"x == 1x ];then
            Pre_mkpart_Disk=$Cache_Disk
            Sector_Optimal
        else
            Pre_mkpart_Disk=$Cache_Disk
            Sector_Optimal
            for i in {1..$Data_Disk_Number};do
                Part_Size="$Block_DB_Size"
                Sector_let
                Part_Number=$((Part_Number + 1))
                Part_Size="$Block_WAL_Size"
                Sector_let
                Part_Number=$((Part_Number + 1))
                if [ "$Enable_BCache"x == yesx ];then
                    Part_Size="$BCache_Size"
                    Sector_let
                    Part_Number=$((Part_Number + 1))
                fi
            done
        fi          

        Part_Size="$Block_DB_Size"
        Sector_let
        Part_Number=$((Part_Number + 1))
        Block_DB_Part_Path=$Cache_Disk_Path-part$Part_Number
        Block_DB_Part="$Cache_Disk$Part_Number"
        Block_DB_Part_Start_Sector="$Part_Start_Sector"
        Block_DB_Part_End_Sector="$Part_End_Sector"

        Part_Size="$Block_WAL_Size"
        Sector_let
        Part_Number=$((Part_Number + 1))
        Block_WAL_Part_Path=$Cache_Disk_Path-part$Part_Number
        Block_WAL_Part="$Cache_Disk$Part_Number"
        Block_WAL_Part_Start_Sector="$Part_Start_Sector"
        Block_WAL_Part_End_Sector="$Part_End_Sector"

        if [ "$Enable_BCache"x == yesx ];then
            Part_Size="$BCache_Size"
            Sector_let
            Part_Number=$((Part_Number + 1))
            Block_BCache_Part_Path=$Cache_Disk_Path-part$Part_Number
            Block_BCache_Part="$Cache_Disk$Part_Number"
            Block_BCache_Part_Start_Sector="$Part_Start_Sector"
            Block_BCache_Part_End_Sector="$Part_End_Sector"
        fi

        #生成osd disk SCSI相关变量
        Cache_Disk_SCSI_ID=`ls -l /sys/block/$Cache_Disk/device|awk -F'/' '{print $NF}'`
        Data_Disk_SCSI_ID=`ls -l /sys/block/$Data_Disk/device|awk -F'/' '{print $NF}'`

        #将osd映射表重定向至文件
        cache_osd_env
    else
        echo "Error: The value Cache is error!"
        exit 1
    fi
}

#generate environment
########################

#creating workflow
###############################################################################

###############################################################################
#destroy workflow
set_osd_out(){
    ceph osd out osd.$OSD_ID
    sleep 5s
}

stop_osd_service(){
    if [ "$Deploy_Mode"x == "binaryx" ];then
        systemctl stop ceph-osd@$OSD_ID
    elif OSD_Process=`ps -ef|egrep -w "^ceph [0-9]* [0-9]* [0-9]* .* /usr/bin/ceph-osd -f --cluster ceph --id $OSD_ID --setuser ceph --setgroup ceph$"`;then
        OSD_Process_Len=`echo "$OSD_Process"|wc -l`
        if [ "$OSD_Process_Len"x != "1x" ];then
            echo "Matches to multiple ceph-osd processes:"
            echo "$OSD_Process"
            exit 1
        elif [ "$OSD_Process_Len"x == "0x" ];then
            echo "The ceph-osd processes is not found."
	else
            OSD_Process_ID=`echo $OSD_Process|awk '{print $2}'`
            kill -9 $OSD_Process_ID
        fi
    fi
}

set_osd_down(){
    ceph osd down osd.$OSD_ID
}

osd_crush_remove(){
    ceph osd crush remove $OSD_ID
}

osd_auth_remove(){
    ceph auth del osd.$OSD_ID
}

osd_rm(){
    ceph osd rm $OSD_ID
}

destroy_bcache_device(){
    BCache_Device=`ls -l /dev/bcache/by-uuid/$BCache_Data_UUID|egrep -w "^.*/bcache[0-9]+$"|awk -F'/' '{print $NF}'`
    if [ -f /sys/fs/bcache/$BCache_Info_Cset_UUID/bdev0/detach ];then
        echo "$BCache_Info_Cset_UUID" > /sys/fs/bcache/$BCache_Info_Cset_UUID/bdev0/detach
    fi
    if [ -f /sys/fs/bcache/$BCache_Info_Cset_UUID/stop ];then
        echo 1 > /sys/fs/bcache/$BCache_Info_Cset_UUID/stop
    fi
    if [ -f /sys/fs/bcache/$BCache_Info_Cset_UUID/unregister ];then
        echo 1 > /sys/fs/bcache/$BCache_Info_Cset_UUID/unregister
    fi
    if [ -f /sys/block/$BCache_Device/bcache/stop ];then
        echo 1 > /sys/block/$BCache_Device/bcache/stop
    fi
    if [ "$BCache_Role"x == cachex ];then
        if [ -f /sys/block/$Cache_Disk/$Block_BCache_Part/bcache/stop ];then
            echo 1 > /sys/block/$Cache_Disk/$Block_BCache_Part/bcache/stop
        fi
        if [ -f /sys/block/$Cache_Disk/$Block_BCache_Part/bcache/unregister ];then
            echo 1 > /sys/block/$Cache_Disk/$Block_BCache_Part/bcache/unregister
        fi
    fi
    if [ -f /sys/block/$Data_Disk/bcache/stop ];then
        echo 1 > /sys/block/$Data_Disk/bcache/stop
    fi
    if [ -f /sys/block/$Data_Disk/bcache/unregister ];then
        echo 1 > /sys/block/$Data_Disk/bcache/unregister
    fi
    sleep 10s
}

remove_osd_from_ceph_cluster(){
    stop_osd_service
    set_osd_down
    set_osd_out
    osd_crush_remove
    osd_auth_remove
    osd_rm
}
#destroy workflow
###############################################################################

###############################################################################
#environment check
osd_device_path_values_check(){
    Devices_Path_Len=
    if [ "$Cache"x == yesx ];then
        echo "Cache_Disk_Path:$Cache_Disk_Path"
        echo "Data_Disk_Path:$Data_Disk_Path"
        echo "Block_DB_Part_Path:$Block_DB_Part_Path"
        echo "Block_WAL_Part_Path:$Block_WAL_Part_Path"
        Disk_Devices_Path="$Data_Disk_Path $Cache_Disk_Path"
        Part_Devices_Path="$Block_DB_Part_Path $Block_WAL_Part_Path"
        Devices_Path="$Disk_Devices_Path $Part_Devices_Path"
        if [ "$Enable_BCache"x == yesx ];then
            if ! test $BCache_Data_UUID;then
                echo "BCache_Data_UUID is null!"
                exit 1
            fi
            echo "Block_BCache_Part_Path:$Block_BCache_Part_Path"
            Part_Devices_Path="$Part_Devices_Path $Block_BCache_Part_Path"
            Devices_Path="$Disk_Devices_Path $Part_Devices_Path"
            for i in $Devices_Path;do
                Devices_Path_Len=$((Devices_Path_Len+1))
            done
            if [ "$Devices_Path_Len"x != 5x ];then
                exit 1
            fi
        else

            for i in $Devices_Path;do
                Devices_Path_Len=$((Devices_Path_Len+1))
            done
            if [ "$Devices_Path_Len"x != 4x ];then
                exit 1
            fi
        fi
    else
        echo "Data_Disk_Path:$Data_Disk_Path"
        echo "Block_DB_Part_Path:$Block_DB_Part_Path"
        echo "Block_WAL_Part_Path:$Block_WAL_Part_Path"

        Devices_Path="$Data_Disk_Path $Block_DB_Part_Path $Block_WAL_Part_Path $Block_Data_Part_Path"
        for i in $Devices_Path;do
            Devices_Path_Len=$((Devices_Path_Len+1))
        done
        if [ "$Devices_Path_Len"x != 4x ];then
            exit 1
        fi
    fi
}

osd_device_values_check(){
    Devices_Len=
    if [ "$Cache"x == yesx ];then
        echo "Cache_Disk:$Cache_Disk"
        echo "Data_Disk:$Data_Disk"
        echo "Block_DB_Part:$Block_DB_Part"
        echo "Block_WAL_Part:$Block_WAL_Part"
        Blue_Devices="/dev/$Block_DB_Part /dev/$Block_WAL_Part /dev/$Data_Disk"
        Disk_Devices="$Data_Disk $Cache_Disk"
        Part_Devices="$Block_DB_Part $Block_WAL_Part"
        Devices="$Disk_Devices $Part_Devices"
        if [ "$Enable_BCache"x == yesx ];then
            echo "Block_BCache_Part:$Block_BCache_Part"
            echo "BCache_Device:$BCache_Device"
            Blue_Devices="/dev/$Block_DB_Part /dev/$Block_WAL_Part /dev/$BCache_Device"
            Disk_Devices="$Data_Disk $Cache_Disk"
            Part_Devices="$Block_DB_Part $Block_WAL_Part $Block_BCache_Part"
            Devices="$Disk_Devices $Part_Devices $BCache_Device"
            for i in $Devices;do
                Devices_Len=$((Devices_Len+1))
            done
            if [ "$Devices_Len"x != 6x ];then
                exit 1
            fi
        else
            for i in $Devices;do
                Devices_Len=$((Devices_Len+1))
            done
            if [ "$Devices_Len"x != 4x ];then
                exit 1
            fi
        fi
    else
        echo "Data_Disk:$Data_Disk"
        echo "Block_DB_Part:$Block_DB_Part"
        echo "Block_WAL_Part:$Block_WAL_Part"
        echo "Block_Data_Part:$Block_Data_Part"
        Blue_Devices="/dev/$Block_DB_Part /dev/$Block_WAL_Part /dev/$Block_Data_Part"
        Disk_Devices="$Data_Disk"
        Part_Devices="$Block_DB_Part $Block_WAL_Part $Block_Data_Part"
        Devices="$Disk_Devices $Part_Devices"
        for i in $Devices;do
            Devices_Len=$((Devices_Len+1))
        done
        if [ "$Devices_Len"x != 3x ];then
            exit 1
        fi
    fi
}

check_osd_status() {
    if ps -aux|egrep -w "^ceph *[0-9]+.*[0-9]+ [0-9]+.*/usr/bin/ceph-osd -f --cluster ceph --id $OSD_ID --setuser ceph --setgroup ceph$";then
        echo "Error: The osd.$OSD_ID process is running"
        exit 1
    fi
}
check_ceph_status() {
    while ! ceph --connect-timeout 1 -s ;do
        echo "ceph cluster is health ok"
    done
}

osd_env_init(){
    create_ceph_user
    create_ceph_confpath
    create_ceph_workpath
    get_ceph_conf
    get_client_admin_keyring
    get_bootstrap_osd_keyring
    check_ceph_status
    check_osd_status
    if [ "$Enable_BCache"x == yesx ];then
        load_bcache_module
    fi
    if ! Ceph_FSID=`ceph --cluster $Cluster_Name fsid`;then
        echo "Error: The ceph cluster fsid get failed!"
        exit 1
    fi
    if ! OSD_Info=`ceph osd info $OSD_ID 2>&1`;then
        echo "Error: The ceph osd is not found!"
    fi
    if ! OSD_Info_Status=`echo $OSD_Info|egrep -w "^osd.$OSD_ID .*"|awk '{print $2}'`;then
        echo "Error: The ceph osd status get failed!"
        exit 1
    fi
    if echo $OSD_Info |grep -w 'does not exist';then
        if [ -f /var/lib/ceph/osd-env/osd.$OSD_ID ];then
            echo "The osd env file osd.$OSD_ID is already exists."
            source /var/lib/ceph/osd-env/osd.$OSD_ID
            osd_device_path_values_check
            osd_device_values_check
            if [ "$OSD_Status"x == destroyx ];then
                remove_osd_from_ceph_cluster
            fi
        else
            echo "The osd env file osd.$OSD_ID is not found."
            echo "generating......"
            if [ "$Deploy_Mode"x == k8sx ];then
                echo "The osd env file osd.$OSD_ID is not found."
                exit 1
            else
                gen_osd_env
                case in $OSD_DEF_OPS;
                    create)
                        echo 'OSD_Status=creating' >> /var/lib/ceph/osd-env/osd.$OSD_ID
                        ;;
                    recovery)
                        echo 'OSD_Status=recovery' >> /var/lib/ceph/osd-env/osd.$OSD_ID
                        ;;
                    rebuild)
                        echo 'OSD_Status=rebuild' >> /var/lib/ceph/osd-env/osd.$OSD_ID
                        ;;
                    destroy)
                        echo 'OSD_Status=destroy' >> /var/lib/ceph/osd-env/osd.$OSD_ID
                        ;;
                    *)
                        echo 'OSD_Status=ready' >> /var/lib/ceph/osd-env/osd.$OSD_ID
                        ;;
                esac
                source /var/lib/ceph/osd-env/osd.$OSD_ID
                osd_device_path_values_check
                osd_device_values_check
            fi
        fi
    elif [ "$OSD_Info_Status"x == upx ];then
        echo "Error: the osd.$OSD_ID is up"
        exit 1
    else
        if [ -f /var/lib/ceph/osd-env/osd.$OSD_ID ];then
            echo "The osd env file osd.$OSD_ID is already exists."
            source /var/lib/ceph/osd-env/osd.$OSD_ID
            osd_device_path_values_check
            osd_device_values_check
            if [ "$OSD_Status"x == destroyx ];then
                remove_osd_from_ceph_cluster
            fi
        else
            echo "The osd env file osd.$OSD_ID is not found."
            echo "generating......"
            gen_osd_env
            source /var/lib/ceph/osd-env/osd.$OSD_ID
            osd_device_path_values_check
            osd_device_values_check
        fi
    fi
}

bcache_check(){
    #check bcachefs
    #check bcache cache uuid
    #input Device_Path=
    if BCache_Info=`bcache-super-show $Device_Path 2>&1` ;then
        BCache_Info_UUID=`echo "$BCache_Info"|grep -w "^dev.uuid"|awk '{print $NF}'`
        BCache_Info_Magic=`echo "$BCache_Info"|grep -w "^sb.magic"|awk '{print $NF}'`
        BCache_Info_Cset_UUID=`echo "$BCache_Info"|grep -w "^cset.uuid"|awk '{print $NF}'`
        BCache_Info_Role=`echo "$BCache_Info"|grep -w "^sb.version"|awk '{print $3}'|awk -F'[' '{print $NF}'`
        case $BCache_Info_Role in
            cache)
                if [ "$BCache_Info_UUID"x == "$BCache_Cache_UUID"x -a "$BCache_Info_Magic"x == okx ];then
                    BCache_Check=done
                else
                    BCache_Cache_Remapped=notfound
                    BCache_Check=notfound
                fi
                ;;
            backing)
                if [ "$BCache_Info_UUID"x == "$BCache_Cache_UUID"x -a "$BCache_Info_Cset_UUID"x == "$BCache_Cset_UUID"x -a "$BCache_Info_Magic"x == okx ];then
                    echo "BCache_Info_UUID $BCache_Info_UUID"
                    BCache_Check=done
                    if [ "$BCache_Backing_Remapping"x == yesx ];then
                        sed -i "/Data_Disk_Path/d" /var/lib/ceph/osd-env/osd.$OSD_ID
                        echo "Data_Disk_Path=$Device_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
                        sed -i "/BCache_Data_UUID/d" /var/lib/ceph/osd-env/osd.$OSD_ID
                        echo "BCache_Data_UUID=$BCache_Info_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID
                        sed -i "/BCache_Cset_UUID/d" /var/lib/ceph/osd-env/osd.$OSD_ID
                        BCache_Info_Cset_UUID=`bcache-super-show $Data_Disk_Path 2>&1|grep -w "^cset.uuid"|awk '{print $NF}'`
                        echo "BCache_Cset_UUID=$BCache_Info_Cset_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID
                        BCache_Backing_Remapped=done
                    fi
                else
                    BCache_Backing_Remapped=notfound
                    BCache_Check=error
                fi
                ;;
            *)
                BCache_Check=error
                ;;
            esac
        BCache_Yes=yes
    else
        BCache_Check=error
        BCache_Yes=no
    fi
    #output BCache_Part_Check= BCache_Yes= BCache_Backing_Remapped=
}

bcache_backing_remapped(){
    Disk_Paths=`ls /dev/disk/by-path/|egrep "hd*|nvme*|sd*|vd*|xvd*|bcache*|cciss*|dasd*|ubd*|ubi*|scm*|pmem*|nbd*|zd*"|awk -F'/' '{print $9}'`
    if ! test $Disks;then
        echo '$Disks is null'
        exit 1
    fi
    BCache_Backing_Remapped=
    for disk_path in $Disk_Paths;do
        BCache_Backing_Remapping=yes
        Device_Path="/dev/disk/by-path/$disk_path"
        bcache_check
        if [ "$Bcache_Backing_Remapped"x == donex ];then
            echo "bcache backing device check done."
            break
        fi
    done
    unset BCache_Backing_Remapping
    #output BCache_Backing_Remapped=
}

blue_role_check(){
    Blue_Info=`ceph-bluestore-tool show-label --dev /dev/$Blue_Device 2>&1`
    Blue_Info_OSD_UUID=`echo "$Blue_Info_OSD_UUID"|grep -w '"osd_uuid"'|awk -F'"' '{print $4}'`
    Device_Blue_Role=`echo "$Blue_Info"|grep '"description"'|awk -F'"' '{print $4}'|tr -s ' ' '_'`
    Blue_Device_Path="/dev/disk/by-path/`ls -l /dev/disk/by-path/|egrep -w "$disk$"|awk '{print $9}'`"
    Blue_Part="-`echo $Blue_Device_Path|awk -F'-' '{print $NF}'`"
    Disk_Path=`if echo $Blue_Device_Path|grep 'part';then echo $Blue_Device_Path |sed -n "s/$Blue_Part$//gp";else echo $Blue_Device_Path;fi`
    if [ "$Blue_Info_OSD_UUID"x == "$OSD_UUID"x -a  "$Blue_Role"x == "$Device_Blue_Role"x ];then
        case $Device_Blue_Role in
            blue_db)
                if [ "$Blue_Remapping"x == yesx ];then
                    sed -i "/Block_DB_Part_Path/d" /var/lib/ceph/osd-env/osd.$OSD_ID
                    echo "Block_DB_Part_Path=$Blue_Device_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
                    if [ "$Cache"x == yesx ];then
                        if ! cat /var/lib/ceph/osd-env/osd.$OSD_ID|egrep -w "^Cache_Disk_Path"|grep -w "$Disk_Path";then
                            sed -i "/Cache_Disk_Path/d" /var/lib/ceph/osd-env/osd.$OSD_ID
                            echo "Cache_Disk_Path=$Disk_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
                        fi
                    else
                        if ! cat /var/lib/ceph/osd-env/osd.$OSD_ID|egrep -w "^Data_Disk_Path"|grep -w "$Disk_Path";then
                            sed -i "/Data_Disk_Path/d" /var/lib/ceph/osd-env/osd.$OSD_ID
                            echo "Data_Disk_Path=$Disk_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
                        fi
                    fi
                    Blue_Remapped="done"
                    Blue_Check='done'
                else
                    Blue_Check='done'
                fi
                ;;
            blue_wal)
                if [ "$Blue_Remapping"x == yesx ];then
                    sed -i "/Block_WAL_Part_Path/d" /var/lib/ceph/osd-env/osd.$OSD_ID
                    echo "Block_WAL_Part_Path=$Blue_Device_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
                    if [ "$Cache"x == yesx ];then
                        if ! cat /var/lib/ceph/osd-env/osd.$OSD_ID|egrep -w "^Cache_Disk_Path"|grep -w "$Disk_Path";then
                            sed -i "/Cache_Disk_Path/d" /var/lib/ceph/osd-env/osd.$OSD_ID
                            echo "Cache_Disk_Path=$Disk_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
                        fi
                    else
                        if ! cat /var/lib/ceph/osd-env/osd.$OSD_ID|egrep -w "^Data_Disk_Path"|grep -w "$Disk_Path";then
                            sed -i "/Data_Disk_Path/d" /var/lib/ceph/osd-env/osd.$OSD_ID
                            echo "Data_Disk_Path=$Disk_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
                        fi
                    fi
                    Blue_Remapped="done"
                    Blue_Check='done'
                else
                    Blue_Check='done'
                fi
                ;;
            main)
                if [ "$Blue_Remapping"x == yesx ];then
                    if [ "$Enable_BCache"x == yesx ];then
                        sed -i "/Block_BCache_Part_Path/d" /var/lib/ceph/osd-env/osd.$OSD_ID
                        echo "Block_BCache_Part_Path=$Blue_Device_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
                    elif [ "$Cache"x == yesx ];then
                        sed -i "/Data_Disk_Path/d" /var/lib/ceph/osd-env/osd.$OSD_ID
                        echo "Data_Disk_Path=$Blue_Device_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
                    else
                        sed -i "/Block_Data_Part_Path/d" /var/lib/ceph/osd-env/osd.$OSD_ID
                        echo "Block_Data_Part_Path=$Blue_Device_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
			if ! cat /var/lib/ceph/osd-env/osd.$OSD_ID|egrep -w "^Data_Disk_Path"|grep -w "$Disk_Path";then
                            sed -i "/Data_Disk_Path/d" /var/lib/ceph/osd-env/osd.$OSD_ID
                            echo "Data_Disk_Path=$Disk_Path" >> /var/lib/ceph/osd-env/osd.$OSD_ID
                        fi
                    fi
                    Blue_Remapped="done"
                    Blue_Check='done'
                else
                    Blue_Check='done'
                fi
                ;;
            *)
               Blue_Check='error'
               ;;
        esac
    else
        Blue_Check='error'
    fi
}

blue_device_remapped(){
    if [ "$Blue_Role"x == blue_dbx -o "$Blue_Role"x == blue_walx ];then
        Disks=`blkid|grep -w "PARTUUID"|grep -v 'bcache'|awk '{print $1}'|tr -s ':' ' '|awk -F'/' '{print $NF}'`
    elif [ "$Enable_BCache"x == yesx ];then
        Disks=`blkid|grep -w "PARTUUID"|grep -w 'bcache'|awk '{print $1}'|tr -s ':' ' '|awk -F'/' '{print $NF}'`
    elif [ "$Cache"x == yesx ];then
        Disks=`blkid|egrep -w "UUID=\"[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*\" TYPE=\".*\"$"|grep -v PART|awk '{print $1}'|tr -s ':' ' '|awk -F'/' '{print $NF}'`
    else
        Disks=`blkid|grep -w "PARTUUID"|grep -v 'bcache'|awk '{print $1}'|tr -s ':' ' '|awk -F'/' '{print $NF}'`
    fi
    #Disks=`ls /dev/disk/by-path/|grep -v total|awk -F'/' '{print $NF}'`
    if ! test $Disks;then
        echo '$Disks is null'
        exit 1
    fi
    Blue_Remapped=
    for disk in $Disks;do
        if echo $disk| egrep "hd*|nvme*|sd*|vd*|xvd*|bcache*|cciss*|dasd*|ubd*|ubi*|scm*|pmem*|nbd*|zd*";then
            Blue_Remapping=yes
            Blue_Device=$disk
            blue_role_check
            if [ "$Blue_Remapped"x == donex ];then
                echo "bluestore data device check done."
                break
            fi
        else
            echo "disk:$disk"
        fi
    done
    if [ "$Blue_Remapped"x != donex ];then
        echo "bluestore data device check error, osd.$OSD_ID bluestore data device not found."
        Blue_Remapped=notfound
    fi
    unset Blue_Remapping
}

#osd_device_values_check
disk_partition_table_check(){
    #input Disk_Path=
    Disk_Info=`parted -s $Disk_Path unit s print 2>&1`
    #判断当前硬盘是否是未创建分区表的空盘
    if echo "$Disk_Part_Info"|grep -w "Error: .*: unrecognised disk label";then
        Device_Path=$Disk_Path
        Disk_Partition_Table_Check=no
    else
        Part_Info=`echo "$Disk_Info"|egrep -w "^[[:space:]][0-9]+[[:space:]]+[0-9]+s[[:space:]]+[0-9]+s[[:space:]]+[0-9]+s"`
        if test $Part_Info;then
            Disk_Parts=`echo "$Part_Info"|wc -l`
        else
            Disk_Parts=0
        fi
        Disk_Partition_Table_Check=yes
    fi
    #output Disk_Partition_Table_Check= Disk_Parts=
}

block_db_part_path_check(){
    #check deploy mode for fusion
    Blue_Role='blue_db'
    if [ -L $Block_DB_Part_Path ];then
        #When the device is present
        Block_DB_Part=`ls -l $Block_DB_Part_Path|awk -F'/' '{print $NF}'`
        Blue_Info=`ceph-bluestore-tool show-label --dev /dev/$Block_DB_Part 2>&1`
        Blue_Info_OSD_UUID=`echo "$Blue_Info"|grep -w '"osd_uuid"'|awk -F'"' '{print $4}'`
        Blue_Info_Role=`echo "$Blue_Info"|grep -w '"description"'|awk -F'"' '{print $4}'`
        Blue_Device=$Block_DB_Part
        Blue_Check=
        if Block_DB_Part_Info_UUID=`blkid /dev/$Block_DB_Part|awk '{print $NF}'|awk -F'"' '{print $2}'|egrep -w "[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*")`;then
            if [ "$Block_DB_Part_UUID"x == "$Block_DB_Part_Info_UUID"x ];then
                Block_DB_Part_UUID_Check=done
            else
                Block_DB_Part_UUID_Check=error
            fi
        else
            Block_DB_Part_UUID_Check=error
        fi
        ##########################
        #check bluestore device role
        if [ "$Blue_Info_OSD_UUID"x == "$OSD_UUID"x -a "$Blue_Role"x == "$Blue_Info_Role"x ];then
            Blue_Check=done
        else
            blue_device_remapped
            #print $Blue_Remapped value:done or error
        fi
        #print $Block_DB_Part_Path_Check  value:done or error
        ##########################

        #judgment current workflow
        if [ "$OSD_Status"x == readyx ];then
            #start osd workflow
            if [ "$Blue_Remapped"x == donex ];then
                osd_env_init
                Block_DB_Part=`ls -l $Block_DB_Part_Path|awk -F'/' '{print $NF}'`
                Block_DB_Part_Path_Check=done
                Block_DB_Part_Remapped=yes
            elif [ "$Blue_Check"x == donex ];then
                Block_DB_Part_Path_Check=done
            else
                Block_DB_Part_Path_Check=error
            fi
        elif [ "$OSD_Status"x == rebuildx -o "$OSD_Status"x == destroyx ];then
            #rebuild osd workflow
            if [ "$Block_DB_Part_UUID_Check"x == donex ];then
                OSD_Device="$Block_DB_Part_Path"
                wipe_osd_device
                Block_DB_Part_Path_Check=done
            else
                if [ "$Blue_Remapped"x == donex ];then
                    osd_env_init
                    Block_DB_Part=`ls -l $Block_DB_Part_Path|awk -F'/' '{print $NF}'`
                    OSD_Device="$Block_DB_Part_Path"
                    wipe_osd_device
                    Block_DB_Part_Path_Check=done
                    Block_DB_Part_Remapped=yes
                elif [ "$Blue_Check"x == donex ];then
                    OSD_Device="$Block_DB_Part_Path"
                    wipe_osd_device
                    Block_DB_Part_Path_Check=done
                else
                    if echo $Block_DB_Part_Path|grep part ;then
                        Device_Path=$Block_DB_Part_Path
                        bcache_check
		        if (echo $Blue_Info_OSD_UUID|egrep -w "[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*") || [ "$BCache_Yes"x == yesx ];then
                            Block_DB_Part_Path_Check=error
                        else
                            OSD_Device="$Block_DB_Part_Path"
                            wipe_osd_device
                            Block_DB_Part_Path_Check=done
                        fi
                    else
                        Block_DB_Part_Path_Check=notfound
                    fi
                fi
            fi
        elif [ "$OSD_Status"x == creatingx ];then
            #create osd workflow
            if [ "$Block_DB_Part_UUID_Check"x == donex ];then
                OSD_Device="$Block_DB_Part_Path"
                wipe_osd_device
                Block_DB_Part_Path_Check=done
            else
                if echo $Block_DB_Part_Path|grep part ;then
                    Device_Path=$Block_DB_Part_Path
                    bcache_check
                    if (echo $Blue_Info_OSD_UUID|egrep -w "[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*") || [ "$BCache_Yes"x == yesx ];then
                        Block_DB_Part_Path_Check=error
                    else
                        OSD_Device="$Block_DB_Part_Path"
                        wipe_osd_device
                        Block_DB_Part_Path_Check=done
                    fi
                else
                    Block_DB_Part_Path_Check=error
                fi
            fi
        else
            #Not found current workflow
            echo "Error: Not found current workflow"
            Block_DB_Part_Path_Check=error
        fi
    else
        #When the device is not found
        if [ "$OSD_Status"x == readyx ];then
            #start osd workflow
            blue_device_remapped
            if [ "$Blue_Check"x == donex ];then
                osd_env_init
                Block_DB_Part=`ls -l $Block_DB_Part_Path|awk -F'/' '{print $NF}'`
                Block_DB_Part_Path_Check=done
            else
                Block_DB_Part_Path_Check=error
            fi
        else
            #create rebuild osd workflow
            Block_DB_Part_Path_Check=notfound
        fi
    fi
}

block_wal_part_path_check(){
    #check deploy mode for fusion
    Blue_Role='blue_wal'
    if [ -L $Block_WAL_Part_Path ];then
        #When the device is present
        Block_WAL_Part=`ls -l $Block_WAL_Part_Path|awk -F'/' '{print $NF}'`
        Blue_Info=`ceph-bluestore-tool show-label --dev /dev/$Block_WAL_Part 2>&1`
        Blue_Info_OSD_UUID=`echo "$Blue_Info"|grep -w '"osd_uuid"'|awk -F'"' '{print $4}'`
        Blue_Info_Role=`echo "$Blue_Info"|grep -w '"description"'|awk -F'"' '{print $4}'`
        Blue_Device=$Block_WAL_Part
        Blue_Check=
        if Block_WAL_Part_Info_UUID=`blkid /dev/$Block_WAL_Part|awk '{print $NF}'|awk -F'"' '{print $2}'|egrep -w "[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*")`;then
            if [ "$Block_WAL_Part_UUID"x == "$Block_WAL_Part_Info_UUID"x ];then
                Block_WAL_Part_UUID_Check=done
            else
                Block_WAL_Part_UUID_Check=error
            fi
        else
            Block_WAL_Part_UUID_Check=error
        fi
        ##########################
        #check bluestore device role
        if [ "$Blue_Info_OSD_UUID"x == "$OSD_UUID"x -a "$Blue_Role"x == "$Blue_Info_Role"x ];then
            Blue_Check=done
        else
            blue_device_remapped
            #print $Blue_Remapped value:done or error
        fi
        #print $Block_WAL_Part_Path_Check  value:done or error
        ##########################

        #judgment current workflow
        if [ "$OSD_Status"x == readyx ];then
            #start osd workflow
            if [ "$Blue_Remapped"x == donex ];then
                osd_env_init
                Block_WAL_Part=`ls -l $Block_WAL_Part_Path|awk -F'/' '{print $NF}'`
                Block_WAL_Part_Path_Check=done
                Block_WAL_Part_Remapped=yes
            elif [ "$Blue_Check"x == donex ];then
                Block_WAL_Part_Path_Check=done
            else
                Block_WAL_Part_Path_Check=error
            fi
        elif [ "$OSD_Status"x == rebuildx -o "$OSD_Status"x == destroyx ];then
            #rebuild osd workflow
            if [ "$Block_WAL_Part_UUID_Check"x == donex ];then
                OSD_Device="$Block_WAL_Part_Path"
                wipe_osd_device
                Block_WAL_Part_Path_Check=done
            else
                if [ "$Blue_Remapped"x == donex ];then
                    osd_env_init
                    Block_WAL_Part=`ls -l $Block_WAL_Part_Path|awk -F'/' '{print $NF}'`
                    OSD_Device="$Block_WAL_Part_Path"
                    wipe_osd_device
                    Block_WAL_Part_Path_Check=done
                    Block_WAL_Part_Remapped=yes
                elif [ "$Blue_Check"x == donex ];then
                    OSD_Device="$Block_WAL_Part_Path"
                    wipe_osd_device
                    Block_WAL_Part_Path_Check=done
                else
                    if echo $Block_WAL_Part_Path|grep part ;then
                        Device_Path=$Block_WAL_Part_Path
                        bcache_check
		        if (echo $Blue_Info_OSD_UUID|egrep -w "[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*") || [ "$BCache_Yes"x == yesx ];then
                            Block_WAL_Part_Path_Check=error
                        else
                            OSD_Device="$Block_WAL_Part_Path"
                            wipe_osd_device
                            Block_WAL_Part_Path_Check=done
                        fi
                        Block_DB_Part_Path_Check=error                  
                    fi
                fi
            fi
        elif [ "$OSD_Status"x == creatingx ];then
            #create osd workflow
            if [ "$Block_WAL_Part_UUID_Check"x == donex ];then
                OSD_Device="$Block_WAL_Part_Path"
                wipe_osd_device
                Block_WAL_Part_Path_Check=done
            else
                if echo $Block_WAL_Part_Path|grep part ;then
                    Device_Path=$Block_WAL_Part_Path
                    bcache_check
                    if (echo $Blue_Info_OSD_UUID|egrep -w "[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*") || [ "$BCache_Yes"x == yesx ];then
                        Block_WAL_Part_Path_Check=error
                    else
                        OSD_Device="$Block_WAL_Part_Path"
                        wipe_osd_device
                        Block_WAL_Part_Path_Check=done
                    fi
                    Block_DB_Part_Path_Check=error
                fi
            fi
        else
            #Not found current workflow
            echo "Error: Not found current workflow"
            Block_WAL_Part_Path_Check=error
        fi
    else
        #When the device is not found
        if [ "$OSD_Status"x == readyx ];then
            #start osd workflow
            blue_device_remapped
            if [ "$Blue_Check"x == donex ];then
                osd_env_init
                Block_WAL_Part=`ls -l $Block_WAL_Part_Path|awk -F'/' '{print $NF}'`
                Block_WAL_Part_Path_Check=done
            else
                Block_WAL_Part_Path_Check=error
            fi
        else
            #create rebuild osd workflow
            Block_WAL_Part_Path_Check=notfound
        fi
    fi
}

data_disk_check_for_bcache(){
    #check deploy mode for cache mode bcache enabled
    Blue_Role=main
    if [ -L $Data_Disk_Path ];then
        #When the device is present
        Data_Disk=`ls -l $Data_Disk_Path|awk -F'/' '{print $NF}'`
        Blue_Info=`ceph-bluestore-tool show-label --dev /dev/$Data_Disk 2>&1`
        Device_Path=$Data_Disk_Path
        bcache_check
        if [ "$BCache_Check"x == donex ];then
            if [ "$OSD_Status"x == readyx ];then
                if [ "$BCache_Info_Magic"x == okx ];then
                    if echo $Blue_Info|grep -w "unable to read label for";then
                        Data_Disk_Path_Check=done
                    else
                        echo "bcache backing device found blustore filesystem!"
                        Data_Disk_Path_Check=error
                    fi
                else
                    echo "bcachefs is error!"
                    Data_Disk_Path_Check=error
                fi
            elif [ "$OSD_Status"x == creating -o "$OSD_Status"x == destroyx -o "$OSD_Status"x == rebuildx ];then
                if echo $Blue_Info|grep -w "unable to read label for";then
                    BCache_Role=backing
                    destroy_bcache_device
                    OSD_Device="$Data_Disk_Path"
                    wipe_osd_device
                    Data_Disk_Path_Check=done
                else
                    echo "bcache backing device found blustore filesystem!"
                    Data_Disk_Path_Check=error
                fi
            else
                echo "Error: unknown Deploy_Mode"
                Data_Disk_Path_Check=error
            fi
        else
            if [ "$OSD_Status"x == readyx ];then
                bcache_backing_remapped
                if [ "$BCache_Backing_Remapped"x == donex ];then
                    if [ "$BCache_Info_Magic"x == okx ];then
                        osd_env_init
                        Data_Disk=`ls -l $Data_Disk_Path|awk -F'/' '{print $NF}'`
                        Blue_Info=`ceph-bluestore-tool show-label --dev /dev/$Data_Disk 2>&1`
                        if echo $Blue_Info|grep -w "unable to read label for";then
                            Data_Disk_Path_Check=done
                        else
                            echo "bcache backing device found blustore filesystem!"
                            Data_Disk_Path_Check=error
                        fi
                    else
                        echo "bcachefs is error!"
                        Data_Disk_Path_Check=error
                    fi
                else
                    Data_Disk_Path_Check=error
                fi
            elif [ "$OSD_Status"x == destroyx -o "$OSD_Status"x == rebuildx -o "$OSD_Status"x == creating ];then
                if echo $Blue_Info|grep -w "unable to read label for";then
                    BCache_Role=backing
                    destroy_bcache_device
                    OSD_Device="$Data_Disk_Path"
                    wipe_osd_device
                    Data_Disk_Path_Check=done
                else
                    Data_Disk_Path_Check=error
                fi
            else
                Data_Disk_Path_Check=error
            fi
        fi
    else
        if [ "$OSD_Status"x == readyx ];then
            bcache_backing_remapped
            if [ "$BCache_Backing_Remapped"x == donex ];then
                if [ "$BCache_Info_Magic"x == okx ];then
                    osd_env_init
                    Data_Disk=`ls -l $Data_Disk_Path|awk -F'/' '{print $NF}'`
                    Blue_Info=`ceph-bluestore-tool show-label --dev /dev/$Data_Disk 2>&1`
                    if echo $Blue_Info|grep -w "unable to read label for";then
                        Data_Disk_Path_Check=done
                    else
                        echo "bcache backing device found blustore filesystem!"
                        Data_Disk_Path_Check=error
                    fi
                else
                    echo "bcachefs is error!"
                    Data_Disk_Path_Check=error
                fi
            else
                Data_Disk_Path_Check=error
            fi
        else
            Data_Disk_Path_Check=error
        fi
    fi
}

data_disk_check_for_cache_no_bcache(){
    #check deploy mode for cache mode (no bcache)
    Blue_Role=main
    if [ -L $Data_Disk_Path ];then
        #When the device is present
        Data_Disk=`ls -l $Data_Disk_Path|awk -F'/' '{print $NF}'`
        Blue_Info=`ceph-bluestore-tool show-label --dev /dev/$Data_Disk 2>&1`
        Blue_Info_OSD_UUID=`echo "$Blue_Info"|grep -w '"osd_uuid"'|awk -F'"' '{print $4}'`
        Blue_Info_Role=`echo "$Blue_Info"|grep -w '"description"'|awk -F'"' '{print $4}'`
        Blue_Device=$Data_Disk
        Blue_Check=
        ##########################
        #check bluestore device role
        if [ "$Blue_Info_OSD_UUID"x == "$OSD_UUID"x -a "$Blue_Role"x == "$Blue_Info_Role"x ];then
            Blue_Check=done
        else
            blue_device_remapped
            #print $Blue_Remapped value:done or error
        fi
        #print $Data_Disk_Path_Check  value:done or error
        ##########################

        #judgment current workflow
        if [ "$OSD_Status"x == readyx ];then
            #start osd workflow
            if [ "$Blue_Remapped"x == donex ];then
                osd_env_init
                Data_Disk=`ls -l $Data_Disk_Path|awk -F'/' '{print $NF}'`
                Data_Disk_Path_Check=done
            elif [ "$Blue_Check"x == donex ];then
                Data_Disk_Path_Check=done
            else
                Data_Disk_Path_Check=error
            fi
        elif [ "$OSD_Status"x == rebuildx -o "$OSD_Status"x == destroyx ];then
            #rebuild osd workflow
            if [ "$Blue_Remapped"x == donex ];then
                osd_env_init
                Data_Disk=`ls -l $Data_Disk_Path|awk -F'/' '{print $NF}'`
                OSD_Device="$Data_Disk_Path"
                wipe_osd_device
                Data_Disk_Path_Check=done
            elif [ "$Blue_Check"x == donex ];then
                OSD_Device="$Data_Disk_Path"
                wipe_osd_device
                Data_Disk_Path_Check=done
            else
                if echo $Blue_Info_OSD_UUID|egrep -w "[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*";then
                    Data_Disk_Path_Check=error
                else
                    disk_partition_table_check
                    if [ "$Disk_Partition_Table_Check"x == nox -o "$Disk_Parts"x == '0x' ];then
                        OSD_Device="$Data_Disk_Path"
                        wipe_osd_device
                        Data_Disk_Path_Check=done
                    else
                        for i in {1..$Disk_Parts};do
                            Tmp_Part="$Data_Disk$i"
                            if Blue_Info=`ceph-bluestore-tool show-label --dev /dev/$Tmp_Part 2>&1`;then
                                Data_Disk_Path_Check=error
                            else
                                OSD_Device="$Data_Disk_Path"
                                wipe_osd_device
                                Data_Disk_Path_Check=done
                            fi
                        done
                    fi
                fi
            fi
        elif [ "$OSD_Status"x == creatingx ];then
            #create osd workflow
            if echo $Blue_Info_OSD_UUID|egrep -w "[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*";then
                Data_Disk_Path_Check=error
            else
                disk_partition_table_check
                if [ "$Disk_Partition_Table_Check"x == nox -o "$Disk_Parts"x == '0x' ];then
                    OSD_Device="$Data_Disk_Path"
                    wipe_osd_device
                    Data_Disk_Path_Check=done
                else
                    for i in {1..$Disk_Parts};do
                        Tmp_Part="$Data_Disk$i"
                        if Blue_Info=`ceph-bluestore-tool show-label --dev /dev/$Tmp_Part 2>&1`;then
                            Data_Disk_Path_Check=error
                        else
                            OSD_Device="$Data_Disk_Path"
                            wipe_osd_device
                            Data_Disk_Path_Check=done
                        fi
                    done
                fi
            fi
        else
            #无法判断当前工作流
            echo "Error: not found current workflow!"
            Data_Disk_Path_Check=error
        fi
    else
        #When the device is not found
        if [ "$OSD_Status"x == readyx ];then
            #start osd workflow
            blue_device_remapped
            if [ "$Blue_Check"x == donex ];then
                osd_env_init
                Data_Disk=`ls -l $Data_Disk_Path|awk -F'/' '{print $NF}'`
                Data_Disk_Path_Check=done
            else
                Data_Disk_Path_Check=notfound
            fi
        else
            #create or rebuild workflow
            Data_Disk_Path_Check=notfound
        fi
    fi
}

block_data_part_path_check(){
    #check deploy mode for fusion
    Blue_Role=main
    if [ -L $Block_Data_Part_Path ];then
        #When the device is present
        Block_Data_Part=`ls -l $Block_Data_Part_Path|awk -F'/' '{print $NF}'`
        Blue_Info=`ceph-bluestore-tool show-label --dev /dev/$Block_Data_Part 2>&1`
        Blue_Info_OSD_UUID=`echo "$Blue_Info"|grep -w '"osd_uuid"'|awk -F'"' '{print $4}'`
        Blue_Info_Role=`echo "$Blue_Info"|grep -w '"description"'|awk -F'"' '{print $4}'`
        Blue_Device=$Block_Data_Part
        Blue_Check=
        if Block_Data_Part_Info_UUID=`blkid /dev/$Block_Data_Part|awk '{print $NF}'|awk -F'"' '{print $2}'|egrep -w "[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*")`;then
            if [ "$Block_Data_Part_UUID"x == "$Block_Data_Part_Info_UUID"x ];then
                Block_Data_Part_UUID_Check=done
            else
                Block_Data_Part_UUID_Check=error
            fi
        else
            Block_Data_Part_UUID_Check=error
        fi
        ##########################
        #check bluestore device role
        if [ "$Blue_Info_OSD_UUID"x == "$OSD_UUID"x -a "$Blue_Role"x == "$Blue_Info_Role"x ];then
            Blue_Check=done
        else
            blue_device_remapped
            #print $Blue_Remapped value:done or error
        fi
        #print $Block_Data_Part_Path_Check  value:done or error
        ##########################

        #judgment current workflow
        if [ "$OSD_Status"x == readyx ];then
            #start osd workflow
            if [ "$Blue_Remapped"x == donex ];then
                osd_env_init
                Block_Data_Part=`ls -l $Block_Data_Part_Path|awk -F'/' '{print $NF}'`
                Block_Data_Part_Path_Check=done
                Block_Data_Part_Remapped=yes
            elif [ "$Blue_Check"x == donex ];then
                Block_Data_Part_Path_Check=done
            else
                Block_Data_Part_Path_Check=error
            fi
        elif [ "$OSD_Status"x == rebuildx -o "$OSD_Status"x == destroyx ];then
            #rebuild osd workflow
            if [ "$Block_Data_Part_UUID_Check"x == donex ];then
                OSD_Device="$Block_Data_Part_Path"
                wipe_osd_device
                Block_Data_Part_Path_Check=done
            else
                if [ "$Blue_Remapped"x == donex ];then
                    osd_env_init
                    Block_Data_Part=`ls -l $Block_Data_Part_Path|awk -F'/' '{print $NF}'`
                    OSD_Device="$Block_Data_Part_Path"
                    wipe_osd_device
                    Block_Data_Part_Path_Check=done
                    Block_Data_Part_Remapped=yes
                elif [ "$Blue_Check"x == donex ];then
                    OSD_Device="$Block_Data_Part_Path"
                    wipe_osd_device
                    Block_Data_Part_Path_Check=done
                else
                    if echo $Block_Data_Part_Path|grep part ;then
                        Device_Path=$Block_Data_Part_Path
                        bcache_check
		        if (echo $Blue_Info_OSD_UUID|egrep -w "[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*") || [ "$BCache_Yes"x == yesx ];then
                            Block_Data_Part_Path_Check=error
                        else
                            OSD_Device="$Block_Data_Part_Path"
                            wipe_osd_device
                            Block_Data_Part_Path_Check=done
                        fi
                        Block_Data_Part_Path_Check=error
                    fi
                fi
            fi
        elif [ "$OSD_Status"x == creatingx ];then
            #create osd workflow
            if [ "$Block_Data_Part_UUID_Check"x == donex ];then
                OSD_Device="$Block_Data_Part_Path"
                wipe_osd_device
                Block_Data_Part_Path_Check=done
            else
                if echo $Block_Data_Part_Path|grep part ;then
                    Device_Path=$Block_Data_Part_Path
                    bcache_check
                    if (echo $Blue_Info_OSD_UUID|egrep -w "[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*") || [ "$BCache_Yes"x == yesx ];then
                        Block_Data_Part_Path_Check=error
                    else
                        OSD_Device="$Block_Data_Part_Path"
                        wipe_osd_device
                        Block_Data_Part_Path_Check=done
                    fi
                    Block_Data_Part_Path_Check=error
                fi
            fi
        else
            #无法判断当前工作流
            echo "Error: not found current workflow!"
            Block_Data_Part_Path_Check=error
        fi
    else
        #When the device is not found
        if [ "$OSD_Status"x == readyx ];then
            #start osd workflow
            blue_device_remapped
            if [ "$Blue_Check"x == donex ];then
                osd_env_init
                Block_Data_Part=`ls -l $Block_Data_Part_Path|awk -F'/' '{print $NF}'`
                Block_Data_Part_Path_Check=done
            else
                Block_Data_Part_Path_Check=error
            fi
        else
            #create rebuild osd workflow
            Block_Data_Part_Path_Check=notfound
        fi
    fi
}

bcache_cache_uuid_remapped(){
    BCache_Info_UUID=`bcache-super-show $Block_BCache_Part_Path 2>&1|grep -w "^dev.uuid"|awk '{print $NF}'`
    BCache_Info_Cset_UUID=`bcache-super-show $Block_BCache_Part_Path 2>&1|grep -w "^cset.uuid"|awk '{print $NF}'`
    echo "BCache_Cset_UUID=$BCache_Info_Cset_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID
    if [ "$BCache_Cache_UUID"x == "$BCache_Info_UUID"x ];then
        sed -i "/BCache_Cache_UUID/d" /var/lib/ceph/osd-env/osd.$OSD_ID
        echo "BCache_Cache_UUID=$BCache_Info_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID 
    else
        sed -i "/BCache_Cset_UUID/d" /var/lib/ceph/osd-env/osd.$OSD_ID
        echo "BCache_Cset_UUID=$BCache_Info_Cset_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID
        sed -i "/BCache_Cache_UUID/d" /var/lib/ceph/osd-env/osd.$OSD_ID
        echo "BCache_Cache_UUID=$BCache_Info_UUID" >> /var/lib/ceph/osd-env/osd.$OSD_ID
    fi        
}

block_bcache_part_path_check(){
    #check deploy mode for fusion
    Blue_Role=main
    if [ -L $Block_BCache_Part_Path ];then
        #When the device is present
        Block_BCache_Part=`ls -l $Block_BCache_Part_Path|awk -F'/' '{print $NF}'`
        Blue_Info=`ceph-bluestore-tool show-label --dev /dev/$Block_BCache_Part 2>&1`
        Blue_Info_OSD_UUID=`echo "$Blue_Info"|grep -w '"osd_uuid"'|awk -F'"' '{print $4}'`
        Blue_Info_Role=`echo "$Blue_Info"|grep -w '"description"'|awk -F'"' '{print $4}'`
        Blue_Device=$Block_BCache_Part
        Blue_Check=
        ##########################
        #check bluestore device role
        if [ "$Blue_Info_OSD_UUID"x == "$OSD_UUID"x -a "$Blue_Role"x == "$Blue_Info_Role"x ];then
            Blue_Check=done
        else
            blue_device_remapped
            #print $Blue_Remapped value:done or error
        fi  
        #print $Block_BCache_Part_Path_Check  value:done or error
        ##########################

        #judgment current workflow
        if [ "$OSD_Status"x == readyx ];then
            #start osd workflow
            if [ "$Blue_Remapped"x == donex ];then
                osd_env_init
                Block_BCache_Part=`ls -l $Block_BCache_Part_Path|awk -F'/' '{print $NF}'`
                bcache_cache_uuid_remapped
                Block_BCache_Part_Path_Check=done
            elif [ "$Blue_Check"x == donex ];then
                Block_BCache_Part_Path_Check=done
            else
                Block_BCache_Part_Path_Check=error
            fi  
        elif [ "$OSD_Status"x == rebuildx -o "$OSD_Status"x == destroyx ];then
            #rebuild osd workflow
            if [ "$Blue_Remapped"x == donex ];then
                osd_env_init
                Block_BCache_Part=`ls -l $Block_BCache_Part_Path|awk -F'/' '{print $NF}'`
                BCache_Role=cache
                destroy_bcache_device
                OSD_Device="$Block_BCache_Part_Path"
                wipe_osd_device
                Block_BCache_Part_Path_Check=done
            elif [ "$Blue_Check"x == donex ];then
                BCache_Role=cache
                destroy_bcache_device
                OSD_Device="$Block_BCache_Part_Path"
                wipe_osd_device
                Block_BCache_Part_Path_Check=done
            else
                if echo $Blue_Info_OSD_UUID|egrep -w "[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*";then
                    Block_BCache_Part_Path_Check=error
                else
                    BCache_Role=cache
                    destroy_bcache_device
                    OSD_Device="$Block_BCache_Part_Path"
                    wipe_osd_device
                    Block_BCache_Part_Path_Check=done
                fi  
            fi  
        elif [ "$OSD_Status"x == creatingx];then
            #create osd workflow
            if echo $Blue_Info_OSD_UUID|egrep -w "[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*-[A-Za-Z0-9]*";then
                Block_Data_Part_Path_Check=error
            else
                BCache_Role=cache
                destroy_bcache_device
                OSD_Device="$Block_BCache_Part_Path"
                wipe_osd_device
                Block_Data_Part_Path_Check=done
            fi
        else
            #无法判断当前工作流
            echo "Error: not found current workflow!"
            Block_BCache_Part_Path_Check=error
        fi  
    else
        #When the device is not found
        if [ "$OSD_Status"x == readyx ];then
            #start osd workflow
            blue_device_remapped
            if [ "$Blue_Check"x == donex ];then
                osd_env_init
                Block_BCache_Part=`ls -l $Block_BCache_Part_Path|awk -F'/' '{print $NF}'`
                bcache_cache_uuid_remapped
                Block_BCache_Part_Path_Check=done
            else
                Block_BCache_Part_Path_Check=error
            fi  
        else
            #create or rebuild workflow
            Block_BCache_Part_Path_Check=notfound
        fi  
    fi  
    unset Blue_Check
    unset Blue_Remapped
    unset Blue_Remapping

}

cache_disk_path_check(){
    #check deploy mode for cache mode or bcache mode
    if [ -L $Cache_Disk_Path ];then
        #When the device is present
        Cache_Disk=`ls -l $Cache_Disk_Path|awk -F'/' '{print $NF}'`
        Disk_Path="$Cache_Disk_Path"
        disk_partition_table_check
        if [ "$Disk_Partition_Table_Check"x == yesx ];then
            Cache_Disk_Path_Check=done
        else
            Blue_Info=`ceph-bluestore-tool show-label --dev /dev/$Cache_Disk 2>&1`
            if echo "$Blue_Info"|grep -w 'unable to read label for';then
                BCache_Info=`bcache-super-show $Cache_Disk_Path 2>&1`
                if ! BCache_Info=`bcache-super-show $Cache_Disk_Path 2>&1`;then
	            Cache_Disk_Path_Check=done
                else
                    Cache_Disk_Path_Check=error
                fi
            else
                Cache_Disk_Path_Check=error
            fi
        fi
    else
        Cache_Disk_Path_Check=error
    fi
}

data_disk_path_check(){
    #check deploy mode for fusion mode
    if [ -L $Data_Disk_Path ];then
        #When the device is present
        Data_Disk=`ls -l $Data_Disk_Path|awk -F'/' '{print $NF}'`
        Disk_Path="$Data_Disk_Path"
        disk_partition_table_check
        if [ "$Disk_Partition_Table_Check"x == yesx ];then
            Data_Disk_Path_Check=done
        else
            Blue_Info=`ceph-bluestore-tool show-label --dev /dev/$Data_Disk 2>&1`
            if echo "$Blue_Info"|grep -w 'unable to read label for';then
                BCache_Info=`bcache-super-show $Block_BCache_Part_Path 2>&1`
                if ! BCache_Info=`bcache-super-show $Data_Disk_Path 2>&1`;then
                    Data_Disk_Path_Check=done
                else
                    Data_Disk_Path_Check=error
                fi
            else
                Data_Disk_Path_Check=error
            fi
        fi
    else
        Data_Disk_Path_Check=error
    fi
}

blue_device_check(){
    osd_env_init
    #check Block_DB_Part_Path
    block_db_part_path_check
    if [ "$Block_DB_Part_Path_Check"x == donex ];then
        echo 'check done'
    elif [ "$Block_DB_Part_Path_Check"x == notfoundx ];then
        if [ "$OSD_Status"x == readyx ];then
            echo "block_db_part_path_check failed. Block_DB_Part_Path device is not found!"
            exit 1
        else 
            echo "check done"
        fi
    else
        echo "block_db_part_path_check failed. "
        exit 1
    fi
    #check Block_WAL_Part_Path
    block_wal_part_path_check
    if [ "$Block_WAL_Part_Path_Check"x == donex ];then
        echo 'check done'
    elif [ "$Block_WAL_Part_Path_Check"x == notfoundx ];then
        if [ "$OSD_Status"x == readyx ];then
            echo "block_wal_part_path_check failed. Block_WAL_Part_Path device is not found!"
            exit 1
        else
            echo "check done"
        fi
    else
        echo "block_wal_part_path_check failed. "
        exit 1
    fi
    if [ "$Cache"x != yesx ];then
        #for fusion mode
        #check Block_Data_Part_Path
        block_data_part_path_check
        if [ "$Block_Data_Part_Path_Check"x == donex ];then
            echo 'check done'
        elif [ "$Block_Data_Part_Path_Check"x == notfoundx ];then
            if [ "$OSD_Status"x == readyx ];then
                echo "block_data_part_path_check failed. Block_Data_Part_Path device is not found!"
                exit 1
            else
                echo "check done"
            fi
        else
            echo "block_data_part_path_check failed. "
            exit 1
        fi
        #check Data_Disk_Path
        data_disk_path_check
        if [ "$Data_Disk_Path_Check"x == donex ];then
            echo 'check done'
        else
            echo "data_disk_path_check failed. "
            exit 1
        fi
    elif [ "$Enable_BCache"x == yesx ];then
        #for bcache mode
        #check Block_BCache_Part_Path
        block_bcache_part_path_check
        if [ "$Block_BCache_Part_Path_Check"x == donex ];then
            echo 'check done'
        elif [ "$Block_BCache_Part_Path_Check"x == notfoundx ];then
            if [ "$OSD_Status"x == readyx ];then
                echo "block_bcache_part_path_check failed. Block_BCache_Part_Path device is not found!"
                exit 1
            else
                echo "check done"
            fi
        else
            echo "block_bcache_part_path_check failed. "
            exit 1
        fi
        #check Data_Disk_Path
        data_disk_check_for_bcache
        if [ "$Data_Disk_Path_Check"x == donex ];then
            echo 'check done'
        else
            echo "data_disk_path_check failed. "
            exit 1
        fi
        cache_disk_path_check
        if [ "$Cache_Disk_Part_Path_Check"x == donex ];then
            echo 'check done'
        elif [ "$Cache_Disk_Part_Path_Check"x == notfoundx ];then
            if [ "$OSD_Status"x == readyx ];then
                echo "cache_disk_part_path_check failed. Cache_Disk_Part_Path device is not found!"
                exit 1
            else
                echo "check done"
            fi
        else
            echo "cache_disk_part_path_check failed. "
            exit 1
        fi

    elif [ "$Cache"x == yesx ];then
        #for cache mode (no bcache)
        #check Cache_Disk_Part_Path
        cache_disk_path_check
        if [ "$Cache_Disk_Part_Path_Check"x == donex ];then
            echo 'check done'
        elif [ "$Cache_Disk_Part_Path_Check"x == notfoundx ];then
            if [ "$OSD_Status"x == readyx ];then
                echo "cache_disk_part_path_check failed. Cache_Disk_Part_Path device is not found!"
                exit 1
            else
                echo "check done"
            fi
        else
            echo "cache_disk_part_path_check failed. "
            exit 1
        fi
        #check Data_Disk_Path
        data_disk_check_for_cache_no_bcache
        if [ "$Data_Disk_Path_Check"x == donex ];then
            echo 'check done'
        else
            echo "data_disk_path_check failed. "
            exit 1
        fi
    else
        echo "Error: deploy mode define error!"
        exit 1
    fi
}
#environment check
##################################################################################

##################################################################################

osd_start_workflow(){
    optimize_kernel_block_io
    chown_osd_device
    chown_osd_device_path
    mount_osd_pwd_for_systemd
    link_block_data_to_osd_workpath
    link_block_db_to_osd_workpath
    link_block_wal_to_osd_workpath
    chown_ceph_pwd
    start_osd_service

}

osd_creating_workflow(){
    if [ "$Enable_BCache"x == yesx ];then
        if [ "$Block_DB_Part_Path_Check"x == notfoundx -a "$Block_WAL_Part_Path_Check"x == notfoundx -a "$Block_BCache_Part_Path_Check"x == notfoundx ];then
            disk_mkpart_stack
            make_bcache
        fi
    elif [ "$Cache"x == yesx ];then
        if [ "$Block_DB_Part_Path_Check"x == notfoundx -a "$Block_WAL_Part_Path_Check"x == notfoundx ];then
            disk_mkpart_stack
        fi
    else
        if [ "$Block_DB_Part_Path_Check"x == notfoundx -a "$Block_WAL_Part_Path_Check"x == notfoundx -a "$Block_Data_Part_Path_Check"x == donex ];then
            disk_mkpart_stack
        fi
    fi
    optimize_kernel_block_io
    chown_osd_device
    chown_osd_device_path
    copy_mount_unit_to_systemd
    mount_osd_pwd_for_systemd
    link_block_data_to_osd_workpath
    link_block_db_to_osd_workpath
    link_block_wal_to_osd_workpath
    chown_ceph_pwd
    request_new_osd
    bootstrap_osd_get_monmap
    create_osd_keyring
    chown_ceph_pwd
    copy_osd_prestart_scripts
    check_device_stats
    osd_mkfs
    check_osd_data_disk
    add_env_file_to_osd_systemd_unit
    systemctl_daemon_reload
    start_osd_service
}

osd_rebuild_workflow(){
    if [ "$Enable_BCache"x == yesx ];then
        if [ "$Block_DB_Part_Path_Check"x == notfoundx -a "$Block_WAL_Part_Path_Check"x == notfoundx -a "$Block_BCache_Part_Path_Check"x == notfoundx ];then
            disk_mkpart_stack
            make_bcache
        fi
    elif [ "$Cache"x == yesx ];then
        if [ "$Block_DB_Part_Path_Check"x == notfoundx -a "$Block_WAL_Part_Path_Check"x == notfoundx ];then
            disk_mkpart_stack
        fi
    else
        if [ "$Block_DB_Part_Path_Check"x == notfoundx -a "$Block_WAL_Part_Path_Check"x == notfoundx -a "$Block_Data_Part_Path_Check"x == donex ];then
            disk_mkpart_stack
        fi
    fi
    optimize_kernel_block_io
    chown_osd_device
    chown_osd_device_path
    copy_mount_unit_to_systemd
    mount_osd_pwd_for_systemd
    link_block_data_to_osd_workpath
    link_block_db_to_osd_workpath
    link_block_wal_to_osd_workpath
    chown_ceph_pwd
    #request_new_osd
    bootstrap_osd_get_monmap
    create_osd_keyring
    chown_ceph_pwd
    copy_osd_prestart_scripts
    check_device_stats
    osd_mkfs
    check_osd_data_disk
    add_env_file_to_osd_systemd_unit
    systemctl_daemon_reload
    start_osd_service
}

osd_destroy_workflow(){
    if [ "$Enable_BCache"x == yesx ];then
        if [ "$Block_DB_Part_Path_Check"x == donex -a "$Block_WAL_Part_Path_Check"x == donex -a "$Block_BCache_Part_Path_Check"x == donex -a "$Cache_Disk_Path_Check"x == donex  -a "$Data_Disk_Path_Check"x == donex ];then
            rm /var/lib/ceph/osd-env/osd.$OSD_ID
	    echo "destroy osd.$OSD_ID done"
        else
            echo "Error: destroy osd.$OSD_ID failed!"	
            exit 1
        fi
    elif [ "$Cache"x == yesx ];then
        if [ "$Block_DB_Part_Path_Check"x == notfoundx -a "$Block_WAL_Part_Path_Check"x == notfoundx -a "$Cache_Disk_Path_Check"x == donex  -a "$Data_Disk_Path_Check"x == donex ];then
            rm /var/lib/ceph/osd-env/osd.$OSD_ID
            echo "destroy osd.$OSD_ID done"
        else
            echo "Error: destroy osd.$OSD_ID failed!"   
            exit 1
        fi
    else
        if [ "$Block_DB_Part_Path_Check"x == notfoundx -a "$Block_WAL_Part_Path_Check"x == notfoundx -a "$Block_Data_Part_Path_Check"x == donex -a "$Data_Disk_Path_Check"x == donex ];then
            rm /var/lib/ceph/osd-env/osd.$OSD_ID
            echo "destroy osd.$OSD_ID done"
        else
            echo "Error: destroy osd.$OSD_ID failed!"   
            exit 1
        fi
    fi
}

osd_init_workflow(){
    blue_device_check
    #include
    #create ceph user
    #create config path
    #create work path
    case in $OSD_Status;
        ready)
           echo "Enter osd.$OSD_ID start workflow."
           osd_start_workflow
           ;;
        recovery)
            echo "Enter osd.$OSD_ID environment recovery workflow."
            osd_recovery_workflow
            ;;
        rebuild)
            echo "Enter osd.$OSD_ID rebuild workflow."
            osd_rebuild_workflow
            ;;
        creating)
            echo "Enter osd.$OSD_ID creating workflow."
            osd_creating_workflow
            ;;
        destroy)
            echo "Enter osd.$OSD_ID destroy workflow."
            osd_destroy_workflow
            ;;
        *)
            echo "Error: The osd.$OSD_ID notfoud workflow."
            exit 1
            ;;
    esac
}

osd_init_workflow
